software/src/data/versification.py

1"""Lossless word-span mapping from OSHB/MorphGNT to the site's Kougo references.23The OSHB map describes full verses. Explicit, source-checked word boundaries4handle its partial verses. Psalm superscriptions use verse 0, never overwrite 1.5"""6from __future__ import annotations78import json9import re10from collections import defaultdict11from pathlib import Path12from xml.etree import ElementTree1314DATA = Path(__file__).resolve().parents[2] / 'data'15OT_IDS = 'genesis exodus leviticus numbers deuteronomy joshua judges ruth 1samuel 2samuel 1kings 2kings 1chronicles 2chronicles ezra nehemiah esther job psalms proverbs ecclesiastes song_of_solomon isaiah jeremiah lamentations ezekiel daniel hosea joel amos obadiah jonah micah nahum habakkuk zephaniah haggai zechariah malachi'.split()16OSIS = 'Gen Exod Lev Num Deut Josh Judg Ruth 1Sam 2Sam 1Kgs 2Kgs 1Chr 2Chr Ezra Neh Esth Job Ps Prov Eccl Song Isa Jer Lam Ezek Dan Hos Joel Amos Obad Jonah Mic Nah Hab Zeph Hag Zech Mal'.split()17OSIS_BOOK = dict(zip(OSIS, OT_IDS, strict=True))18# (source book, chapter, verse): [(target chapter, verse, start word, end word)]19# Half-open, zero-based word spans; None means the end of the source verse.20PARTIAL = {21    ('1kings', 18, 34): [(18, 33, 0, 10), (18, 34, 10, None)],22    ('1kings', 20, 3): [(20, 2, 0, 6), (20, 3, 6, None)],23    ('1kings', 22, 21): [(22, 21, 0, 8), (22, 22, 8, None)],24    ('1kings', 22, 44): [(22, 43, 0, None)],25    ('isaiah', 63, 19): [(63, 19, 0, 9), (64, 1, 9, None)],26    ('psalms', 13, 6): [(13, 5, 0, 6), (13, 6, 6, None)],27}28BOUNDARIES = {29    ('1kings', 18, 34): (10, 'ויאמר'),30    ('1kings', 20, 3): (6, 'כספך'),31    ('1kings', 22, 21): (8, 'ויאמר'),32    ('isaiah', 63, 19): (9, 'לוא'),33    ('psalms', 13, 6): (6, 'אשירה'),34    ('zechariah', 2, 15): (12, 'וידעת'),35}363738def parse_osis(ref):39    book, chapter, verse = ref.split('!')[0].split('.')40    return OSIS_BOOK[book], int(chapter), int(verse)414243def load_map(path=DATA / 'versification/oshb-VerseMap.xml'):44    mapping = {}45    partial_keys = set()46    for item in ElementTree.parse(path).iter('{http://www.APTBibleTools.com/namespace}verse'):47        source, target = parse_osis(item.attrib['wlc']), parse_osis(item.attrib['kjv'])48        if item.attrib['type'] == 'full':49            if source in mapping:50                raise ValueError(f'Duplicate source mapping: {source}')51            mapping[source] = [(target[1], target[2], 0, None)]52        elif item.attrib['type'] == 'partial':53            partial_keys.add(source)54        else:55            raise ValueError(f'Unknown map type: {item.attrib}')56    if partial_keys != set(PARTIAL):57        raise ValueError('Partial verse map changed; review word boundaries')58    mapping.update(PARTIAL)59    # Hebrew headings precede the source verse mapped to English verse 1.60    for (book, chapter, verse), targets in list(mapping.items()):61        if book == 'psalms' and targets[0][:2] == (chapter, 1):62            for heading in range(1, verse):63                mapping[(book, chapter, heading)] = [(chapter, 0, 0, None)]64    # Greek/SBLGNT numbering differs from the main site's printed units here.65    mapping[('3john', 1, 15)] = [(1, 14, 0, None)]66    mapping[('revelation', 12, 18)] = [(12, 17, 0, None)]67    # Kougo puts the closing recognition formula in 2:12 (KJV has it in 2:11).68    mapping[('zechariah', 2, 15)] = [(2, 11, 0, 12), (2, 12, 12, None)]69    return mapping707172def spans_for(book, chapter, verse, record, mapping):73    words = record['words']74    key = (book, chapter, verse)75    spans = mapping.get(key, [(chapter, verse, 0, None)])76    if key in BOUNDARIES:77        offset, expected = BOUNDARIES[key]78        if offset >= len(words) or re.sub(r'[^א-ת]', '', words[offset]['word']) != expected:79            raise ValueError(f'Source changed at reviewed boundary: {key}')80    if key == ('acts', 19, 40):81        # Six-word closing sentence belongs to Acts 19:41 in Kougo/KJV.82        ending = 'καὶ ταῦτα εἰπὼν ἀπέλυσεν τὴν ἐκκλησίαν'83        if ' '.join(w['word'] for w in words[-6:]) != ending:84            raise ValueError('Acts 19:40 closing sentence changed')85        spans = [(19, 40, 0, len(words)-6), (19, 41, len(words)-6, None)]86    covered = []87    result = []88    for ch, v, start, end in spans:89        stop = len(words) if end is None else end90        if not 0 <= start < stop <= len(words):91            raise ValueError(f'Invalid word span: {key}')92        covered.extend(range(start, stop))93        result.append(((book, ch, v), start, stop))94    if covered != list(range(len(words))):95        raise ValueError(f'Word loss or duplication: {key}')96    return result979899STRONGS_HEAD = re.compile(r'(\d+)')100101102def hebrew_strongs(lemma):103    """The head Strong's number from a morphhb lemma.104105    morphhb lemmas are augmented: prefixes are separated with "/" and some106    carry a homonym letter ("1254 a"). The head word is the last numeric107    segment, which is what a dictionary or a concordance is keyed on.108    """109    parts = [p for p in (lemma or '').split('/') if STRONGS_HEAD.search(p)]110    if not parts:111        return ''112    found = STRONGS_HEAD.search(parts[-1])113    return f'H{int(found.group(1))}' if found else ''114115116def greek_strongs(data_dir):117    """Strong's numbers per Greek verse, in word order.118119    From macula-greek, never from MorphGNT: its lemma and parsing fields are120    CC BY-SA and this project deliberately does not distribute them.121    """122    path = data_dir / 'sources/greek_strongs.json'123    if not path.exists():124        return {}125    return json.loads(path.read_text())['verses']126127128def build_originals(data_dir=DATA):129    mapping = load_map(data_dir / 'versification/oshb-VerseMap.xml')130    greek_keys = greek_strongs(data_dir)131    targets = defaultdict(list)132    source_map = {}133    for language in ('hebrew', 'greek'):134        for path in sorted((data_dir / 'sources' / language).glob('*.json')):135            book = path.stem136            for ref, record in json.loads(path.read_text()).items():137                ch, v = map(int, ref.split(':'))138                source = (book, ch, v)139                source_map[source] = spans_for(book, ch, v, record, mapping)140                for target, start, stop in source_map[source]:141                    # Language is identified per word, including mixed verses.142                    for offset in range(start, stop):143                        word = record['words'][offset]144                        if language == 'hebrew' and not word.get('morph', '').startswith(('H', 'A')):145                            raise ValueError(f'Unknown word language: {source}:{offset}')146                        lang = 'grc' if language == 'greek' else ('arc' if word['morph'].startswith('A') else 'he')147                        text = word['word'].replace('/', '')148                        if language == 'greek':149                            keys = greek_keys.get(f'{book}:{ch}:{v}', [])150                            strong = keys[offset] if offset < len(keys) else ''151                        else:152                            strong = hebrew_strongs(word.get('lemma'))153                        targets[target].append({'language': lang, 'text': text, 'strong': strong,154                                                'source': f'{book}:{ch}:{v}', 'word': offset})155    # Sort by source reference and word position, including merged verses.156    for words in targets.values():157        words.sort(key=lambda w: (*map(int, w['source'].split(':')[1:]), w['word']))158    return dict(targets), source_map159