software/scripts/fetch_macula_strongs.py

1#!/usr/bin/env python32"""Fetch Greek Strong's numbers keyed to SBLGNT, from Clear-Bible/macula-greek.34Deliberately NOT from MorphGNT. MorphGNT's parsing and lemmatisation fields are5CC BY-SA 3.0, and this project's NOTICE records that they are not distributed;6building a Strong's index from its `lemma` column would pull share-alike into7the shipped app. macula-greek is CC BY 4.0 and is keyed word-by-word to SBLGNT8specifically, so it aligns with the text already bundled.910Usage: uv run python scripts/fetch_macula_strongs.py11"""12from __future__ import annotations1314import argparse15import csv16import hashlib17import io18import json19import re20from collections import defaultdict21from pathlib import Path2223import httpx2425SOURCE = ("https://raw.githubusercontent.com/Clear-Bible/macula-greek/main/"26          "SBLGNT/tsv/macula-greek-SBLGNT.tsv")2728# macula uses three-letter ids; this project uses the lowercase English name.29BOOKS = {30    'MAT': 'matthew', 'MRK': 'mark', 'LUK': 'luke', 'JHN': 'john', 'ACT': 'acts',31    'ROM': 'romans', '1CO': '1corinthians', '2CO': '2corinthians', 'GAL': 'galatians',32    'EPH': 'ephesians', 'PHP': 'philippians', 'COL': 'colossians',33    '1TH': '1thessalonians', '2TH': '2thessalonians', '1TI': '1timothy',34    '2TI': '2timothy', 'TIT': 'titus', 'PHM': 'philemon', 'HEB': 'hebrews',35    'JAS': 'james', '1PE': '1peter', '2PE': '2peter', '1JN': '1john',36    '2JN': '2john', '3JN': '3john', 'JUD': 'jude', 'REV': 'revelation',37}38REFERENCE = re.compile(r'([A-Z0-9]{3}) (\d+):(\d+)!(\d+)')394041def parse(text):42    """Strong's numbers per verse, in word order."""43    verses = defaultdict(list)44    for row in csv.DictReader(io.StringIO(text), delimiter='\t'):45        match = REFERENCE.match(row.get('ref') or '')46        if not match:47            continue48        book = BOOKS.get(match.group(1))49        if book is None:50            continue51        number = (row.get('strong') or '').strip()52        verses[f'{book}:{int(match.group(2))}:{int(match.group(3))}'].append(53            f'G{number}' if number else '')54    return dict(verses)555657def build(out_path, text=None):58    if text is None:59        text = httpx.get(SOURCE, timeout=600, follow_redirects=True).raise_for_status().text60    verses = parse(text)61    if len(verses) < 7900:62        raise ValueError(f'Expected the whole Greek New Testament, got {len(verses)} verses')63    payload = json.dumps({'schema': 1, 'source': SOURCE,64                          'source_sha256': hashlib.sha256(text.encode()).hexdigest(),65                          'verses': verses}, ensure_ascii=False, separators=(',', ':')) + '\n'66    out_path.parent.mkdir(parents=True, exist_ok=True)67    out_path.write_text(payload, encoding='utf-8')68    return {'verses': len(verses), 'words': sum(len(v) for v in verses.values())}697071if __name__ == '__main__':72    parser = argparse.ArgumentParser(description=__doc__)73    parser.add_argument('--out', type=Path, default=Path('data/sources/greek_strongs.json'))74    args = parser.parse_args()75    report = build(args.out)76    print(f"Wrote {report['words']} Greek Strong's keys across {report['verses']} verses")77