software/scripts/download_sources.py

1"""Download and parse public domain Bible source texts."""23from __future__ import annotations45import json6import xml.etree.ElementTree as ET7from pathlib import Path89import httpx10from rich.console import Console11from rich.progress import (12    BarColumn,13    MofNCompleteColumn,14    Progress,15    SpinnerColumn,16    TaskProgressColumn,17    TextColumn,18    TimeElapsedColumn,19)20from rich.table import Table2122# Base URLs for source data23MORPHGNT_BASE = "https://raw.githubusercontent.com/morphgnt/sblgnt/master"24MORPHHB_BASE = "https://raw.githubusercontent.com/openscriptures/morphhb/master/wlc"25OPENBIBLE_BASE = "https://raw.githubusercontent.com/churchstudio-org/openbible/main"2627console = Console()2829# NT Book mappings30# morphgnt uses XX-Abbr format, openbible uses name or name# format31NT_BOOKS = {32    "matthew": {"morphgnt": "61-Mt", "openbible": "matthew"},33    "mark": {"morphgnt": "62-Mk", "openbible": "mark"},34    "luke": {"morphgnt": "63-Lk", "openbible": "luke"},35    "john": {"morphgnt": "64-Jn", "openbible": "john"},36    "acts": {"morphgnt": "65-Ac", "openbible": "acts"},37    "romans": {"morphgnt": "66-Ro", "openbible": "romans"},38    "1corinthians": {"morphgnt": "67-1Co", "openbible": "corinthians1"},39    "2corinthians": {"morphgnt": "68-2Co", "openbible": "corinthians2"},40    "galatians": {"morphgnt": "69-Ga", "openbible": "galatians"},41    "ephesians": {"morphgnt": "70-Eph", "openbible": "ephesians"},42    "philippians": {"morphgnt": "71-Php", "openbible": "philippians"},43    "colossians": {"morphgnt": "72-Col", "openbible": "colossians"},44    "1thessalonians": {"morphgnt": "73-1Th", "openbible": "thessalonians1"},45    "2thessalonians": {"morphgnt": "74-2Th", "openbible": "thessalonians2"},46    "1timothy": {"morphgnt": "75-1Ti", "openbible": "timothy1"},47    "2timothy": {"morphgnt": "76-2Ti", "openbible": "timothy2"},48    "titus": {"morphgnt": "77-Tit", "openbible": "titus"},49    "philemon": {"morphgnt": "78-Phm", "openbible": "philemon"},50    "hebrews": {"morphgnt": "79-Heb", "openbible": "hebrews"},51    "james": {"morphgnt": "80-Jas", "openbible": "james"},52    "1peter": {"morphgnt": "81-1Pe", "openbible": "peter1"},53    "2peter": {"morphgnt": "82-2Pe", "openbible": "peter2"},54    "1john": {"morphgnt": "83-1Jn", "openbible": "john1"},55    "2john": {"morphgnt": "84-2Jn", "openbible": "john2"},56    "3john": {"morphgnt": "85-3Jn", "openbible": "john3"},57    "jude": {"morphgnt": "86-Jud", "openbible": "jude"},58    "revelation": {"morphgnt": "87-Re", "openbible": "revelation"},59}6061# OT Book mappings62# morphhb uses book abbreviations, openbible uses full names63OT_BOOKS = {64    "genesis": {"morphhb": "Gen", "openbible": "genesis"},65    "exodus": {"morphhb": "Exod", "openbible": "exodus"},66    "leviticus": {"morphhb": "Lev", "openbible": "leviticus"},67    "numbers": {"morphhb": "Num", "openbible": "numbers"},68    "deuteronomy": {"morphhb": "Deut", "openbible": "deuteronomy"},69    "joshua": {"morphhb": "Josh", "openbible": "joshua"},70    "judges": {"morphhb": "Judg", "openbible": "judges"},71    "ruth": {"morphhb": "Ruth", "openbible": "ruth"},72    "1samuel": {"morphhb": "1Sam", "openbible": "samuel1"},73    "2samuel": {"morphhb": "2Sam", "openbible": "samuel2"},74    "1kings": {"morphhb": "1Kgs", "openbible": "kings1"},75    "2kings": {"morphhb": "2Kgs", "openbible": "kings2"},76    "1chronicles": {"morphhb": "1Chr", "openbible": "chronicles1"},77    "2chronicles": {"morphhb": "2Chr", "openbible": "chronicles2"},78    "ezra": {"morphhb": "Ezra", "openbible": "ezra"},79    "nehemiah": {"morphhb": "Neh", "openbible": "nehemiah"},80    "esther": {"morphhb": "Esth", "openbible": "esther"},81    "job": {"morphhb": "Job", "openbible": "job"},82    "psalms": {"morphhb": "Ps", "openbible": "psalms"},83    "proverbs": {"morphhb": "Prov", "openbible": "proverbs"},84    "ecclesiastes": {"morphhb": "Eccl", "openbible": "ecclesiastes"},85    "song_of_solomon": {"morphhb": "Song", "openbible": "songofsolomon"},86    "isaiah": {"morphhb": "Isa", "openbible": "isaiah"},87    "jeremiah": {"morphhb": "Jer", "openbible": "jeremiah"},88    "lamentations": {"morphhb": "Lam", "openbible": "lamentations"},89    "ezekiel": {"morphhb": "Ezek", "openbible": "ezekiel"},90    "daniel": {"morphhb": "Dan", "openbible": "daniel"},91    "hosea": {"morphhb": "Hos", "openbible": "hosea"},92    "joel": {"morphhb": "Joel", "openbible": "joel"},93    "amos": {"morphhb": "Amos", "openbible": "amos"},94    "obadiah": {"morphhb": "Obad", "openbible": "obadiah"},95    "jonah": {"morphhb": "Jonah", "openbible": "jonah"},96    "micah": {"morphhb": "Mic", "openbible": "micah"},97    "nahum": {"morphhb": "Nah", "openbible": "nahum"},98    "habakkuk": {"morphhb": "Hab", "openbible": "habakkuk"},99    "zephaniah": {"morphhb": "Zeph", "openbible": "zephaniah"},100    "haggai": {"morphhb": "Hag", "openbible": "haggai"},101    "zechariah": {"morphhb": "Zech", "openbible": "zechariah"},102    "malachi": {"morphhb": "Mal", "openbible": "malachi"},103}104105# Combined mapping for convenience106ALL_BOOKS = {**OT_BOOKS, **NT_BOOKS}107108109def download_file(url: str) -> str | None:110    """Download a file from URL."""111    try:112        response = httpx.get(url, timeout=30.0, follow_redirects=True)113        response.raise_for_status()114        return response.text115    except httpx.HTTPError:116        return None117118119def parse_morphgnt_book(content: str) -> dict[str, dict]:120    """Parse MorphGNT space-separated data into verse dictionary.121122    Format: BBCCVV pos parsing text_punct word norm lemma (space-separated)123    where BB=book, CC=chapter, VV=verse124    """125    verses: dict[str, dict] = {}126127    for line in content.strip().split("\n"):128        if not line.strip():129            continue130131        parts = line.split()132        if len(parts) < 7:133            continue134135        # Parse reference (format: BBCCVV where BB=book, CC=chapter, VV=verse)136        ref = parts[0]137        try:138            chapter = int(ref[2:4])139            verse = int(ref[4:6])140        except (ValueError, IndexError):141            continue142143        ref_key = f"{chapter}:{verse}"144145        if ref_key not in verses:146            verses[ref_key] = {"words": [], "text": ""}147148        # Add word data149        # Format: ref pos parsing text_punct word norm lemma150        word_data = {151            "word": parts[4],  # Word without punctuation152            "normalized": parts[5],153            "lemma": parts[6],154            "pos": parts[1],155            "parsing": parts[2],156        }157        verses[ref_key]["words"].append(word_data)158159    # Build full text for each verse160    for ref_key, data in verses.items():161        words = [w["word"] for w in data["words"]]162        data["text"] = " ".join(words)163164    # Convert to simple format: ref -> {text, translit}165    result = {}166    for ref_key, data in verses.items():167        result[ref_key] = {168            "text": data["text"],169            "words": data["words"],170        }171172    return result173174175def parse_morphhb_xml(content: str) -> dict[str, dict]:176    """Parse MorphHB OSIS XML format into verse dictionary.177178    MorphHB uses OSIS XML format with <verse> elements containing <w> word elements.179    """180    verses: dict[str, dict] = {}181182    try:183        # Parse XML184        root = ET.fromstring(content)185    except ET.ParseError:186        return {}187188    # Find all verse elements189    for verse_elem in root.iter(190        "{http://www.bibletechnologies.net/2003/OSIS/namespace}verse"191    ):192        osisID = verse_elem.get("osisID", "")193        if not osisID:194            continue195196        # Parse osisID format: Book.Chapter.Verse197        parts = osisID.split(".")198        if len(parts) < 3:199            continue200201        try:202            chapter = int(parts[1])203            verse = int(parts[2])204        except ValueError:205            continue206207        ref_key = f"{chapter}:{verse}"208209        # Collect words from this verse210        words = []211        word_data_list = []212213        for w_elem in verse_elem.iter(214            "{http://www.bibletechnologies.net/2003/OSIS/namespace}w"215        ):216            word = w_elem.text or ""217            word = word.strip()218            if not word:219                continue220221            lemma = w_elem.get("lemma", "")222            morph = w_elem.get("morph", "")223224            words.append(word)225            word_data_list.append({226                "word": word,227                "lemma": lemma,228                "morph": morph,229            })230231        if words:232            verses[ref_key] = {233                "text": " ".join(words),234                "words": word_data_list,235            }236237    return verses238239240def parse_openbible_json(content: str) -> dict[str, str]:241    """Parse OpenBible JSON format into verse dictionary.242243    OpenBible uses 3D array: bible[book][chapter][verse]244    But individual book files use: book[chapter][verse]245    """246    try:247        data = json.loads(content)248    except json.JSONDecodeError:249        return {}250251    verses = {}252253    # Handle both full bible and individual book formats254    if isinstance(data, list):255        # Individual book: [chapter][verse]256        for chapter_idx, chapter in enumerate(data):257            if not isinstance(chapter, list):258                continue259            for verse_idx, verse_text in enumerate(chapter):260                if verse_text:261                    ref_key = f"{chapter_idx + 1}:{verse_idx + 1}"262                    verses[ref_key] = verse_text263264    return verses265266267def download_greek_nt(output_dir: Path, book: str) -> tuple[bool, int]:268    """Download Greek NT from MorphGNT.269270    Returns:271        Tuple of (success, verse_count)272    """273    if book not in NT_BOOKS:274        return False, 0275276    morphgnt_code = NT_BOOKS[book]["morphgnt"]277    url = f"{MORPHGNT_BASE}/{morphgnt_code}-morphgnt.txt"278279    content = download_file(url)280    if not content:281        return False, 0282283    verses = parse_morphgnt_book(content)284285    # Save to JSON286    output_file = output_dir / "greek" / f"{book}.json"287    output_file.parent.mkdir(parents=True, exist_ok=True)288289    with output_file.open("w", encoding="utf-8") as f:290        json.dump(verses, f, ensure_ascii=False, indent=2)291292    return True, len(verses)293294295def download_hebrew_ot(output_dir: Path, book: str) -> tuple[bool, int]:296    """Download Hebrew OT from MorphHB (Westminster Leningrad Codex).297298    Returns:299        Tuple of (success, verse_count)300    """301    if book not in OT_BOOKS:302        return False, 0303304    morphhb_code = OT_BOOKS[book]["morphhb"]305    url = f"{MORPHHB_BASE}/{morphhb_code}.xml"306307    content = download_file(url)308    if not content:309        return False, 0310311    verses = parse_morphhb_xml(content)312313    # Save to JSON314    output_file = output_dir / "hebrew" / f"{book}.json"315    output_file.parent.mkdir(parents=True, exist_ok=True)316317    with output_file.open("w", encoding="utf-8") as f:318        json.dump(verses, f, ensure_ascii=False, indent=2)319320    return True, len(verses)321322323def download_kjv(output_dir: Path, book: str) -> tuple[bool, int]:324    """Download KJV from OpenBible.325326    Returns:327        Tuple of (success, verse_count)328    """329    if book in NT_BOOKS:330        openbible_name = NT_BOOKS[book]["openbible"]331    elif book in OT_BOOKS:332        openbible_name = OT_BOOKS[book]["openbible"]333    else:334        return False, 0335336    url = f"{OPENBIBLE_BASE}/KJV/books/{openbible_name}.json"337338    content = download_file(url)339    if not content:340        return False, 0341342    verses = parse_openbible_json(content)343344    # Save to JSON345    output_file = output_dir / "english" / "kjv" / f"{book}.json"346    output_file.parent.mkdir(parents=True, exist_ok=True)347348    with output_file.open("w", encoding="utf-8") as f:349        json.dump(verses, f, ensure_ascii=False, indent=2)350351    return True, len(verses)352353354def download_kougo(output_dir: Path, book: str) -> tuple[bool, int]:355    """Download Japanese Kougo-yaku from OpenBible.356357    Returns:358        Tuple of (success, verse_count)359    """360    if book in NT_BOOKS:361        openbible_name = NT_BOOKS[book]["openbible"]362    elif book in OT_BOOKS:363        openbible_name = OT_BOOKS[book]["openbible"]364    else:365        return False, 0366367    url = f"{OPENBIBLE_BASE}/KOUGO_YAKU/books/{openbible_name}.json"368369    content = download_file(url)370    if not content:371        return False, 0372373    verses = parse_openbible_json(content)374375    # Save to Japanese kougo directory376    output_file = output_dir / "japanese" / "kougo" / f"{book}.json"377    output_file.parent.mkdir(parents=True, exist_ok=True)378379    with output_file.open("w", encoding="utf-8") as f:380        json.dump(verses, f, ensure_ascii=False, indent=2)381382    return True, len(verses)383384385def download_book_with_progress(386    output_dir: Path,387    book: str,388    progress: Progress,389    book_task: int,390) -> dict[str, tuple[bool, int]]:391    """Download all sources for a book with progress updates.392393    Returns:394        Dictionary of source type to (success, verse_count)395    """396    is_ot = book in OT_BOOKS397    results: dict[str, tuple[bool, int]] = {}398399    # Create sub-tasks for each source type400    if is_ot:401        sources = [402            ("Hebrew", lambda: download_hebrew_ot(output_dir, book)),403            ("KJV", lambda: download_kjv(output_dir, book)),404            ("Kougo", lambda: download_kougo(output_dir, book)),405        ]406    else:407        sources = [408            ("Greek", lambda: download_greek_nt(output_dir, book)),409            ("KJV", lambda: download_kjv(output_dir, book)),410            ("Kougo", lambda: download_kougo(output_dir, book)),411        ]412413    for source_name, download_func in sources:414        progress.update(book_task, description=f"[cyan]{book}[/] - {source_name}")415        success, count = download_func()416        results[source_name.lower()] = (success, count)417        progress.advance(book_task)418419    return results420421422def create_progress() -> Progress:423    """Create a rich Progress instance with custom columns."""424    return Progress(425        SpinnerColumn(),426        TextColumn("[bold blue]{task.description}"),427        BarColumn(bar_width=40),428        TaskProgressColumn(),429        MofNCompleteColumn(),430        TimeElapsedColumn(),431        console=console,432        expand=False,433    )434435436def download_all_books(437    output_dir: Path,438    books: dict[str, dict],439    testament_name: str,440) -> dict[str, dict[str, tuple[bool, int]]]:441    """Download all books with visual progress bars.442443    Args:444        output_dir: Output directory for source files.445        books: Dictionary of book names to their metadata.446        testament_name: "Old Testament" or "New Testament" for display.447448    Returns:449        Dictionary of book name to source results.450    """451    all_results: dict[str, dict[str, tuple[bool, int]]] = {}452453    console.print(f"\n[bold green]{testament_name}[/] ({len(books)} books)")454    console.print("─" * 60)455456    with create_progress() as progress:457        # Overall progress for all books458        overall_task = progress.add_task(459            f"[yellow]Downloading {testament_name}...",460            total=len(books),461        )462463        for book in books:464            # Per-book progress (3 sources each)465            book_task = progress.add_task(466                f"[cyan]{book}[/]",467                total=3,468            )469470            results = download_book_with_progress(471                output_dir, book, progress, book_task472            )473            all_results[book] = results474475            # Mark book task as complete and update overall476            progress.update(book_task, visible=False)477            progress.advance(overall_task)478479    return all_results480481482def print_summary(483    ot_results: dict[str, dict[str, tuple[bool, int]]] | None,484    nt_results: dict[str, dict[str, tuple[bool, int]]] | None,485) -> None:486    """Print a summary table of download results."""487    console.print("\n[bold]Download Summary[/]")488    console.print("=" * 60)489490    table = Table(show_header=True, header_style="bold magenta")491    table.add_column("Testament", style="cyan")492    table.add_column("Books", justify="right")493    table.add_column("Source", justify="right")494    table.add_column("KJV", justify="right")495    table.add_column("Kougo", justify="right")496    table.add_column("Total Verses", justify="right")497498    if ot_results:499        ot_source_ok = sum(1 for r in ot_results.values() if r.get("hebrew", (False, 0))[0])500        ot_kjv_ok = sum(1 for r in ot_results.values() if r.get("kjv", (False, 0))[0])501        ot_kougo_ok = sum(1 for r in ot_results.values() if r.get("kougo", (False, 0))[0])502        ot_verses = sum(r.get("hebrew", (False, 0))[1] for r in ot_results.values())503        table.add_row(504            "Old Testament",505            str(len(ot_results)),506            f"[green]{ot_source_ok}[/] Hebrew",507            f"[green]{ot_kjv_ok}[/]",508            f"[green]{ot_kougo_ok}[/]",509            f"{ot_verses:,}",510        )511512    if nt_results:513        nt_source_ok = sum(1 for r in nt_results.values() if r.get("greek", (False, 0))[0])514        nt_kjv_ok = sum(1 for r in nt_results.values() if r.get("kjv", (False, 0))[0])515        nt_kougo_ok = sum(1 for r in nt_results.values() if r.get("kougo", (False, 0))[0])516        nt_verses = sum(r.get("greek", (False, 0))[1] for r in nt_results.values())517        table.add_row(518            "New Testament",519            str(len(nt_results)),520            f"[green]{nt_source_ok}[/] Greek",521            f"[green]{nt_kjv_ok}[/]",522            f"[green]{nt_kougo_ok}[/]",523            f"{nt_verses:,}",524        )525526    console.print(table)527528    # Total529    total_books = (len(ot_results) if ot_results else 0) + (len(nt_results) if nt_results else 0)530    total_verses = (531        (sum(r.get("hebrew", (False, 0))[1] for r in ot_results.values()) if ot_results else 0) +532        (sum(r.get("greek", (False, 0))[1] for r in nt_results.values()) if nt_results else 0)533    )534    console.print(f"\n[bold green]Total: {total_books} books, {total_verses:,} verses[/]")535536537def download_single_book(output_dir: Path, book: str) -> None:538    """Download a single book with progress display."""539    if book in OT_BOOKS:540        testament = "Old Testament"541        books = {book: OT_BOOKS[book]}542    elif book in NT_BOOKS:543        testament = "New Testament"544        books = {book: NT_BOOKS[book]}545    else:546        console.print(f"[red]Unknown book: {book}[/]")547        return548549    results = download_all_books(output_dir, books, testament)550551    # Show results for single book552    book_results = results.get(book, {})553    console.print(f"\n[bold]{book.title()}[/] download complete:")554    for source, (success, count) in book_results.items():555        status = "[green]OK[/]" if success else "[red]FAILED[/]"556        console.print(f"  {source}: {status} ({count} verses)")557558559def main() -> None:560    """Main entry point."""561    import argparse562563    parser = argparse.ArgumentParser(564        description="Download Bible source texts with progress display"565    )566    parser.add_argument(567        "--book",568        default=None,569        help="Single book to download",570    )571    parser.add_argument(572        "--output",573        default="data/sources",574        help="Output directory (default: data/sources)",575    )576    parser.add_argument(577        "--all-nt",578        action="store_true",579        help="Download all NT books",580    )581    parser.add_argument(582        "--all-ot",583        action="store_true",584        help="Download all OT books",585    )586    parser.add_argument(587        "--all",588        action="store_true",589        help="Download all books (OT + NT)",590    )591592    args = parser.parse_args()593    output_dir = Path(args.output)594595    console.print("[bold]Bible Source Downloader[/]")596    console.print(f"Output directory: {output_dir}")597598    ot_results = None599    nt_results = None600601    if args.all:602        ot_results = download_all_books(output_dir, OT_BOOKS, "Old Testament")603        nt_results = download_all_books(output_dir, NT_BOOKS, "New Testament")604        print_summary(ot_results, nt_results)605    elif args.all_ot:606        ot_results = download_all_books(output_dir, OT_BOOKS, "Old Testament")607        print_summary(ot_results, None)608    elif args.all_nt:609        nt_results = download_all_books(output_dir, NT_BOOKS, "New Testament")610        print_summary(None, nt_results)611    elif args.book:612        download_single_book(output_dir, args.book)613    else:614        # Default: download john615        download_single_book(output_dir, "john")616617618if __name__ == "__main__":619    main()620