"""Build lazily loaded original-language books using the reviewed word mapping.""" from __future__ import annotations import csv import hashlib import json import re from collections import Counter from pathlib import Path from src.data.versification import DATA, build_originals ROOT = DATA.parents[2] WEB = ROOT / 'web/japanbible-net/web' def normalize_book(name): for roman, number in [('III ', '3'), ('II ', '2'), ('I ', '1')]: if name.startswith(roman): name = number + name[len(roman):] break return name.lower().replace(' ', '').replace('revelationofjohn', 'revelation').replace('songofsolomon', 'song_of_solomon') def load_site(web_dir=WEB): names = dict(re.findall(r"'([^']+)': \{ code: '([^']+)'", (web_dir / 'src/lib/data.ts').read_text())) refs = {} with (web_dir / 'public/JapKougo.csv').open(encoding='utf-8', newline='') as f: for row in csv.DictReader(f): key = (normalize_book(row['Book']), int(row['Chapter']), int(row['Verse'])) if key in refs or row['Book'] not in names: raise ValueError(f'Invalid site reference: {key}') refs[key] = row['Text'] codes = {normalize_book(name): code for name, code in names.items()} if len(codes) != 66 or len({k[:2] for k in refs}) != 1189: raise ValueError('Expected a complete 66-book site corpus') return refs, codes def group_segments(words): """Language runs, keeping the words that make them up. `text` stays exactly what it was — the joined run the reader sees — so the rendered scripture is unchanged. `words` is additive, and is what a tapped word needs in order to be looked up at all. """ result = [] keys = [] for word in words: if ' ' in word['text']: raise ValueError(f"Word contains a space, so text cannot be split back: {word}") if result and (result[-1]['language'], result[-1]['source']) == (word['language'], word['source']): result[-1]['text'] += ' ' + word['text'] keys[-1].append(word.get('strong') or '') else: result.append({k: word[k] for k in ('language', 'source', 'text')}) keys.append([word.get('strong') or '']) # `text` is space-joined and no word contains a space, so the words are # recoverable by splitting it. Only the keys need storing, which costs a # few bytes a word instead of repeating the whole text a second time. for segment, strongs in zip(result, keys): if any(strongs): segment['strongs'] = ' '.join(strongs) return result def build(out_dir, data_dir=DATA, web_dir=WEB): out_dir = Path(out_dir).resolve() # This exporter must never overwrite its inputs or the translation output. if out_dir.is_relative_to(data_dir.resolve()): raise ValueError('Export destination must be outside translator data') refs, codes = load_site(web_dir) originals, source_map = build_originals(data_dir) unexpected = [k for k in originals if k[2] and k not in refs] if unexpected: raise ValueError(f'Original references outside site: {unexpected}') books = {book: {'schema': 1, 'book': code, 'chapters': {}} for book, code in codes.items()} missing = [] for book, chapter, verse in sorted(refs): ch = books[book]['chapters'].setdefault(str(chapter), {'heading': [], 'verses': {}}) words = originals.get((book, chapter, verse), []) ch['verses'][str(verse)] = group_segments(words) if not words: missing.append(f'{book}:{chapter}:{verse}') for (book, chapter, verse), words in originals.items(): if verse == 0: books[book]['chapters'][str(chapter)]['heading'] = group_segments(words) files = {} out_dir.mkdir(parents=True, exist_ok=True) for book, payload in books.items(): text = json.dumps(payload, ensure_ascii=False, separators=(',', ':')) + '\n' filename = codes[book] + '.json' (out_dir / filename).write_text(text, encoding='utf-8') files[filename] = hashlib.sha256(text.encode()).hexdigest() summary = {'schema': 1, 'source_references': len(source_map), 'site_references': len(refs), 'mapped_source_references': sum(any(t != k for t, _, _ in spans) for k, spans in source_map.items()), 'heading_chapters': sum(k[2] == 0 for k in originals), 'words_by_language': dict(Counter(w['language'] for ws in originals.values() for w in ws)), 'without_original': missing, 'files': files, 'source_files': {str(p.relative_to(data_dir)): hashlib.sha256(p.read_bytes()).hexdigest() for language in ('hebrew', 'greek') for p in sorted((data_dir/'sources'/language).glob('*.json'))}, 'map_sha256': hashlib.sha256((data_dir/'versification/oshb-VerseMap.xml').read_bytes()).hexdigest()} (out_dir/'manifest.json').write_text(json.dumps(summary, ensure_ascii=False, indent=2)+'\n') return summary