software/scripts/fetch_pericope_data.py

1#!/usr/bin/env python32"""3Fetch and extract pericope/paragraph data from SBLGNT.45This script downloads the SBLGNT OSIS XML and extracts paragraph6boundaries to augment the manually curated pericopes.json.78Usage:9    uv run python scripts/fetch_pericope_data.py [--book BOOK]1011Data Sources:12    - SBLGNT OSIS XML: https://sblgnt.com/download/sblgnt.osis.zip13    - KJV with paragraph markers: https://github.com/farskipper/kjv14"""1516from __future__ import annotations1718import argparse19import json20import re21import tempfile22import zipfile23from pathlib import Path24from urllib.request import urlretrieve25from xml.etree import ElementTree as ET2627# OSIS XML namespace28OSIS_NS = {"osis": "http://www.bibletechnologies.net/2003/OSIS/namespace"}2930# Book name mappings (OSIS ID -> our format)31BOOK_MAP = {32    "Matt": "matthew",33    "Mark": "mark",34    "Luke": "luke",35    "John": "john",36    "Acts": "acts",37    "Rom": "romans",38    "1Cor": "1corinthians",39    "2Cor": "2corinthians",40    "Gal": "galatians",41    "Eph": "ephesians",42    "Phil": "philippians",43    "Col": "colossians",44    "1Thess": "1thessalonians",45    "2Thess": "2thessalonians",46    "1Tim": "1timothy",47    "2Tim": "2timothy",48    "Titus": "titus",49    "Phlm": "philemon",50    "Heb": "hebrews",51    "Jas": "james",52    "1Pet": "1peter",53    "2Pet": "2peter",54    "1John": "1john",55    "2John": "2john",56    "3John": "3john",57    "Jude": "jude",58    "Rev": "revelation",59}606162def download_sblgnt_osis(cache_dir: Path) -> Path:63    """Download SBLGNT OSIS XML if not cached."""64    zip_path = cache_dir / "sblgnt.osis.zip"65    xml_path = cache_dir / "sblgnt.osis.xml"6667    if xml_path.exists():68        print(f"Using cached: {xml_path}")69        return xml_path7071    print("Downloading SBLGNT OSIS XML...")72    url = "https://sblgnt.com/download/sblgnt.osis.zip"7374    cache_dir.mkdir(parents=True, exist_ok=True)75    urlretrieve(url, zip_path)7677    print("Extracting...")78    with zipfile.ZipFile(zip_path, "r") as zf:79        # Find the XML file in the zip80        xml_files = [n for n in zf.namelist() if n.endswith(".xml")]81        if xml_files:82            zf.extract(xml_files[0], cache_dir)83            extracted = cache_dir / xml_files[0]84            extracted.rename(xml_path)8586    return xml_path878889def extract_paragraphs_from_osis(xml_path: Path, book_filter: str | None = None) -> dict:90    """91    Extract paragraph boundaries from SBLGNT OSIS XML.9293    The SBLGNT uses <p> elements to mark paragraph breaks.94    Verse references are in <verse> elements with osisID attributes.9596    Returns:97        Dictionary of book -> list of paragraph boundaries98    """99    print(f"Parsing {xml_path}...")100    tree = ET.parse(xml_path)101    root = tree.getroot()102103    paragraphs = {}104105    # Find all div elements (books)106    for div in root.findall(".//osis:div[@type='book']", OSIS_NS):107        book_id = div.get("osisID", "")108        book_name = BOOK_MAP.get(book_id, book_id.lower())109110        if book_filter and book_name != book_filter:111            continue112113        print(f"Processing {book_name}...")114        book_paragraphs = []115        current_para_verses = []116        current_para_start = None117118        # Iterate through all elements in order119        for elem in div.iter():120            tag = elem.tag.replace(f"{{{OSIS_NS['osis']}}}", "")121122            if tag == "p":123                # New paragraph - save previous if exists124                if current_para_verses:125                    book_paragraphs.append({126                        "start": current_para_start,127                        "end": current_para_verses[-1],128                        "verses": current_para_verses.copy(),129                    })130                current_para_verses = []131                current_para_start = None132133            elif tag == "verse":134                osis_id = elem.get("osisID", "")135                if osis_id and "." in osis_id:136                    # Parse "John.1.1" format137                    parts = osis_id.split(".")138                    if len(parts) >= 3:139                        chapter = int(parts[1])140                        verse = int(parts[2])141                        verse_ref = f"{chapter}:{verse}"142143                        if current_para_start is None:144                            current_para_start = verse_ref145                        current_para_verses.append(verse_ref)146147        # Don't forget the last paragraph148        if current_para_verses:149            book_paragraphs.append({150                "start": current_para_start,151                "end": current_para_verses[-1],152                "verses": current_para_verses,153            })154155        if book_paragraphs:156            paragraphs[book_name] = book_paragraphs157158    return paragraphs159160161def merge_with_pericopes(162    paragraphs: dict,163    pericopes_path: Path,164) -> dict:165    """166    Merge extracted paragraph data with existing pericopes.json.167168    Adds paragraph boundaries as a reference while preserving169    manually curated pericope divisions.170    """171    if pericopes_path.exists():172        with pericopes_path.open("r", encoding="utf-8") as f:173            pericopes = json.load(f)174    else:175        pericopes = {176            "version": "1.0",177            "source": "SBLGNT paragraph extraction",178            "books": {},179        }180181    # Add paragraph data to each book182    for book_name, book_paras in paragraphs.items():183        if book_name not in pericopes["books"]:184            pericopes["books"][book_name] = {185                "name": book_name.title(),186                "pericopes": [],187            }188189        # Store raw paragraph data separately190        pericopes["books"][book_name]["sblgnt_paragraphs"] = book_paras191192    return pericopes193194195def generate_pericopes_from_paragraphs(paragraphs: list[dict], book: str) -> list[dict]:196    """197    Generate pericope entries from paragraph data.198199    Combines small paragraphs into larger units (target: 5-15 verses).200    """201    pericopes = []202    current_verses = []203    pericope_num = 1204205    for para in paragraphs:206        current_verses.extend(para["verses"])207208        # If we have enough verses, create a pericope209        if len(current_verses) >= 8 or para == paragraphs[-1]:210            if current_verses:211                start = current_verses[0]212                end = current_verses[-1]213                start_ch, start_v = map(int, start.split(":"))214                end_ch, end_v = map(int, end.split(":"))215216                pericopes.append({217                    "id": f"{book}-{start_ch}-{start_v}-auto",218                    "title": None,219                    "title_ja": None,220                    "start_chapter": start_ch,221                    "start_verse": start_v,222                    "end_chapter": end_ch,223                    "end_verse": end_v,224                    "verses": current_verses.copy(),225                    "theme": None,226                    "auto_generated": True,227                })228                current_verses = []229                pericope_num += 1230231    return pericopes232233234def generate_paragraphs_from_sources(book: str, sources_dir: Path) -> list[dict]:235    """Fallback paragraph generation from local source texts.236237    The SBLGNT OSIS download is the textual apparatus (commentary238    annotations), not the running text — it contains no verse elements, so239    paragraph extraction yields nothing. And OT books are not in SBLGNT at240    all. This fallback windows the verses of each chapter (from the local241    greek/hebrew source JSON) into fixed-size pseudo-paragraphs so that242    chunk/pericope translation is available for every book.243    """244    for lang in ("greek", "hebrew"):245        path = sources_dir / lang / f"{book}.json"246        if path.exists():247            with path.open("r", encoding="utf-8") as f:248                data = json.load(f)249            break250    else:251        return []252253    by_chapter: dict[int, list[int]] = {}254    for key in data:255        parts = key.split(":")256        if len(parts) == 2:257            by_chapter.setdefault(int(parts[0]), []).append(int(parts[1]))258259    paragraphs = []260    window = 6261    for chapter in sorted(by_chapter):262        verses = sorted(by_chapter[chapter])263        for i in range(0, len(verses), window):264            chunk = verses[i : i + window]265            refs = [f"{chapter}:{v}" for v in chunk]266            paragraphs.append({"start": refs[0], "end": refs[-1], "verses": refs})267268    return paragraphs269270271def main():272    parser = argparse.ArgumentParser(273        description="Fetch and extract pericope data from SBLGNT"274    )275    parser.add_argument(276        "--book",277        help="Only process specific book (e.g., 'john')",278    )279    parser.add_argument(280        "--output",281        default="data/metadata/pericopes.json",282        help="Output file path",283    )284    parser.add_argument(285        "--cache-dir",286        default=".cache/sblgnt",287        help="Cache directory for downloaded files",288    )289    parser.add_argument(290        "--auto-generate",291        action="store_true",292        help="Auto-generate pericopes from paragraphs",293    )294    args = parser.parse_args()295296    # Get project root297    script_dir = Path(__file__).parent298    project_root = script_dir.parent299    cache_dir = project_root / args.cache_dir300    output_path = project_root / args.output301302    # Download SBLGNT303    xml_path = download_sblgnt_osis(cache_dir)304305    # Extract paragraphs306    paragraphs = extract_paragraphs_from_osis(xml_path, args.book)307308    print(f"\nExtracted paragraphs from {len(paragraphs)} books:")309    for book, paras in paragraphs.items():310        print(f"  {book}: {len(paras)} paragraphs")311312    # Fallback: OSIS extraction found nothing for the requested book313    # (apparatus-only download, or an OT book not in SBLGNT).314    if args.book and args.book not in paragraphs:315        fallback = generate_paragraphs_from_sources(316            args.book, project_root / "data" / "sources"317        )318        if fallback:319            print(320                f"OSIS had no paragraphs for {args.book}; generated "321                f"{len(fallback)} windowed pseudo-paragraphs from local sources"322            )323            paragraphs[args.book] = fallback324325    # Merge with existing pericopes326    result = merge_with_pericopes(paragraphs, output_path)327328    # Optionally auto-generate pericopes329    if args.auto_generate:330        for book_name, book_paras in paragraphs.items():331            if book_name in result["books"]:332                existing = result["books"][book_name].get("pericopes", [])333                if not existing:334                    print(f"Auto-generating pericopes for {book_name}...")335                    result["books"][book_name]["pericopes"] = (336                        generate_pericopes_from_paragraphs(book_paras, book_name)337                    )338339    # Save340    output_path.parent.mkdir(parents=True, exist_ok=True)341    with output_path.open("w", encoding="utf-8") as f:342        json.dump(result, f, ensure_ascii=False, indent=2)343344    print(f"\nSaved to: {output_path}")345346347if __name__ == "__main__":348    main()349