software/src/output/originals.py

1"""Build lazily loaded original-language books using the reviewed word mapping."""2from __future__ import annotations34import csv5import hashlib6import json7import re8from collections import Counter9from pathlib import Path1011from src.data.versification import DATA, build_originals1213ROOT = DATA.parents[2]14WEB = ROOT / 'web/japanbible-net/web'151617def normalize_book(name):18    for roman, number in [('III ', '3'), ('II ', '2'), ('I ', '1')]:19        if name.startswith(roman):20            name = number + name[len(roman):]21            break22    return name.lower().replace(' ', '').replace('revelationofjohn', 'revelation').replace('songofsolomon', 'song_of_solomon')232425def load_site(web_dir=WEB):26    names = dict(re.findall(r"'([^']+)': \{ code: '([^']+)'", (web_dir / 'src/lib/data.ts').read_text()))27    refs = {}28    with (web_dir / 'public/JapKougo.csv').open(encoding='utf-8', newline='') as f:29        for row in csv.DictReader(f):30            key = (normalize_book(row['Book']), int(row['Chapter']), int(row['Verse']))31            if key in refs or row['Book'] not in names:32                raise ValueError(f'Invalid site reference: {key}')33            refs[key] = row['Text']34    codes = {normalize_book(name): code for name, code in names.items()}35    if len(codes) != 66 or len({k[:2] for k in refs}) != 1189:36        raise ValueError('Expected a complete 66-book site corpus')37    return refs, codes383940def group_segments(words):41    """Language runs, keeping the words that make them up.4243    `text` stays exactly what it was — the joined run the reader sees — so the44    rendered scripture is unchanged. `words` is additive, and is what a tapped45    word needs in order to be looked up at all.46    """47    result = []48    keys = []49    for word in words:50        if ' ' in word['text']:51            raise ValueError(f"Word contains a space, so text cannot be split back: {word}")52        if result and (result[-1]['language'], result[-1]['source']) == (word['language'], word['source']):53            result[-1]['text'] += ' ' + word['text']54            keys[-1].append(word.get('strong') or '')55        else:56            result.append({k: word[k] for k in ('language', 'source', 'text')})57            keys.append([word.get('strong') or ''])58    # `text` is space-joined and no word contains a space, so the words are59    # recoverable by splitting it. Only the keys need storing, which costs a60    # few bytes a word instead of repeating the whole text a second time.61    for segment, strongs in zip(result, keys):62        if any(strongs):63            segment['strongs'] = ' '.join(strongs)64    return result656667def build(out_dir, data_dir=DATA, web_dir=WEB):68    out_dir = Path(out_dir).resolve()69    # This exporter must never overwrite its inputs or the translation output.70    if out_dir.is_relative_to(data_dir.resolve()):71        raise ValueError('Export destination must be outside translator data')72    refs, codes = load_site(web_dir)73    originals, source_map = build_originals(data_dir)74    unexpected = [k for k in originals if k[2] and k not in refs]75    if unexpected:76        raise ValueError(f'Original references outside site: {unexpected}')77    books = {book: {'schema': 1, 'book': code, 'chapters': {}} for book, code in codes.items()}78    missing = []79    for book, chapter, verse in sorted(refs):80        ch = books[book]['chapters'].setdefault(str(chapter), {'heading': [], 'verses': {}})81        words = originals.get((book, chapter, verse), [])82        ch['verses'][str(verse)] = group_segments(words)83        if not words:84            missing.append(f'{book}:{chapter}:{verse}')85    for (book, chapter, verse), words in originals.items():86        if verse == 0:87            books[book]['chapters'][str(chapter)]['heading'] = group_segments(words)88    files = {}89    out_dir.mkdir(parents=True, exist_ok=True)90    for book, payload in books.items():91        text = json.dumps(payload, ensure_ascii=False, separators=(',', ':')) + '\n'92        filename = codes[book] + '.json'93        (out_dir / filename).write_text(text, encoding='utf-8')94        files[filename] = hashlib.sha256(text.encode()).hexdigest()95    summary = {'schema': 1, 'source_references': len(source_map),96               'site_references': len(refs), 'mapped_source_references': sum(any(t != k for t, _, _ in spans) for k, spans in source_map.items()),97               'heading_chapters': sum(k[2] == 0 for k in originals),98               'words_by_language': dict(Counter(w['language'] for ws in originals.values() for w in ws)),99               'without_original': missing, 'files': files,100               'source_files': {str(p.relative_to(data_dir)): hashlib.sha256(p.read_bytes()).hexdigest() for language in ('hebrew', 'greek') for p in sorted((data_dir/'sources'/language).glob('*.json'))},101               'map_sha256': hashlib.sha256((data_dir/'versification/oshb-VerseMap.xml').read_bytes()).hexdigest()}102    (out_dir/'manifest.json').write_text(json.dumps(summary, ensure_ascii=False, indent=2)+'\n')103    return summary104