"""Download and parse public domain Bible source texts.""" from __future__ import annotations import json import xml.etree.ElementTree as ET from pathlib import Path import httpx from rich.console import Console from rich.progress import ( BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn, ) from rich.table import Table # Base URLs for source data MORPHGNT_BASE = "https://raw.githubusercontent.com/morphgnt/sblgnt/master" MORPHHB_BASE = "https://raw.githubusercontent.com/openscriptures/morphhb/master/wlc" OPENBIBLE_BASE = "https://raw.githubusercontent.com/churchstudio-org/openbible/main" console = Console() # NT Book mappings # morphgnt uses XX-Abbr format, openbible uses name or name# format NT_BOOKS = { "matthew": {"morphgnt": "61-Mt", "openbible": "matthew"}, "mark": {"morphgnt": "62-Mk", "openbible": "mark"}, "luke": {"morphgnt": "63-Lk", "openbible": "luke"}, "john": {"morphgnt": "64-Jn", "openbible": "john"}, "acts": {"morphgnt": "65-Ac", "openbible": "acts"}, "romans": {"morphgnt": "66-Ro", "openbible": "romans"}, "1corinthians": {"morphgnt": "67-1Co", "openbible": "corinthians1"}, "2corinthians": {"morphgnt": "68-2Co", "openbible": "corinthians2"}, "galatians": {"morphgnt": "69-Ga", "openbible": "galatians"}, "ephesians": {"morphgnt": "70-Eph", "openbible": "ephesians"}, "philippians": {"morphgnt": "71-Php", "openbible": "philippians"}, "colossians": {"morphgnt": "72-Col", "openbible": "colossians"}, "1thessalonians": {"morphgnt": "73-1Th", "openbible": "thessalonians1"}, "2thessalonians": {"morphgnt": "74-2Th", "openbible": "thessalonians2"}, "1timothy": {"morphgnt": "75-1Ti", "openbible": "timothy1"}, "2timothy": {"morphgnt": "76-2Ti", "openbible": "timothy2"}, "titus": {"morphgnt": "77-Tit", "openbible": "titus"}, "philemon": {"morphgnt": "78-Phm", "openbible": "philemon"}, "hebrews": {"morphgnt": "79-Heb", "openbible": "hebrews"}, "james": {"morphgnt": "80-Jas", "openbible": "james"}, "1peter": {"morphgnt": "81-1Pe", "openbible": "peter1"}, "2peter": {"morphgnt": "82-2Pe", "openbible": "peter2"}, "1john": {"morphgnt": "83-1Jn", "openbible": "john1"}, "2john": {"morphgnt": "84-2Jn", "openbible": "john2"}, "3john": {"morphgnt": "85-3Jn", "openbible": "john3"}, "jude": {"morphgnt": "86-Jud", "openbible": "jude"}, "revelation": {"morphgnt": "87-Re", "openbible": "revelation"}, } # OT Book mappings # morphhb uses book abbreviations, openbible uses full names OT_BOOKS = { "genesis": {"morphhb": "Gen", "openbible": "genesis"}, "exodus": {"morphhb": "Exod", "openbible": "exodus"}, "leviticus": {"morphhb": "Lev", "openbible": "leviticus"}, "numbers": {"morphhb": "Num", "openbible": "numbers"}, "deuteronomy": {"morphhb": "Deut", "openbible": "deuteronomy"}, "joshua": {"morphhb": "Josh", "openbible": "joshua"}, "judges": {"morphhb": "Judg", "openbible": "judges"}, "ruth": {"morphhb": "Ruth", "openbible": "ruth"}, "1samuel": {"morphhb": "1Sam", "openbible": "samuel1"}, "2samuel": {"morphhb": "2Sam", "openbible": "samuel2"}, "1kings": {"morphhb": "1Kgs", "openbible": "kings1"}, "2kings": {"morphhb": "2Kgs", "openbible": "kings2"}, "1chronicles": {"morphhb": "1Chr", "openbible": "chronicles1"}, "2chronicles": {"morphhb": "2Chr", "openbible": "chronicles2"}, "ezra": {"morphhb": "Ezra", "openbible": "ezra"}, "nehemiah": {"morphhb": "Neh", "openbible": "nehemiah"}, "esther": {"morphhb": "Esth", "openbible": "esther"}, "job": {"morphhb": "Job", "openbible": "job"}, "psalms": {"morphhb": "Ps", "openbible": "psalms"}, "proverbs": {"morphhb": "Prov", "openbible": "proverbs"}, "ecclesiastes": {"morphhb": "Eccl", "openbible": "ecclesiastes"}, "song_of_solomon": {"morphhb": "Song", "openbible": "songofsolomon"}, "isaiah": {"morphhb": "Isa", "openbible": "isaiah"}, "jeremiah": {"morphhb": "Jer", "openbible": "jeremiah"}, "lamentations": {"morphhb": "Lam", "openbible": "lamentations"}, "ezekiel": {"morphhb": "Ezek", "openbible": "ezekiel"}, "daniel": {"morphhb": "Dan", "openbible": "daniel"}, "hosea": {"morphhb": "Hos", "openbible": "hosea"}, "joel": {"morphhb": "Joel", "openbible": "joel"}, "amos": {"morphhb": "Amos", "openbible": "amos"}, "obadiah": {"morphhb": "Obad", "openbible": "obadiah"}, "jonah": {"morphhb": "Jonah", "openbible": "jonah"}, "micah": {"morphhb": "Mic", "openbible": "micah"}, "nahum": {"morphhb": "Nah", "openbible": "nahum"}, "habakkuk": {"morphhb": "Hab", "openbible": "habakkuk"}, "zephaniah": {"morphhb": "Zeph", "openbible": "zephaniah"}, "haggai": {"morphhb": "Hag", "openbible": "haggai"}, "zechariah": {"morphhb": "Zech", "openbible": "zechariah"}, "malachi": {"morphhb": "Mal", "openbible": "malachi"}, } # Combined mapping for convenience ALL_BOOKS = {**OT_BOOKS, **NT_BOOKS} def download_file(url: str) -> str | None: """Download a file from URL.""" try: response = httpx.get(url, timeout=30.0, follow_redirects=True) response.raise_for_status() return response.text except httpx.HTTPError: return None def parse_morphgnt_book(content: str) -> dict[str, dict]: """Parse MorphGNT space-separated data into verse dictionary. Format: BBCCVV pos parsing text_punct word norm lemma (space-separated) where BB=book, CC=chapter, VV=verse """ verses: dict[str, dict] = {} for line in content.strip().split("\n"): if not line.strip(): continue parts = line.split() if len(parts) < 7: continue # Parse reference (format: BBCCVV where BB=book, CC=chapter, VV=verse) ref = parts[0] try: chapter = int(ref[2:4]) verse = int(ref[4:6]) except (ValueError, IndexError): continue ref_key = f"{chapter}:{verse}" if ref_key not in verses: verses[ref_key] = {"words": [], "text": ""} # Add word data # Format: ref pos parsing text_punct word norm lemma word_data = { "word": parts[4], # Word without punctuation "normalized": parts[5], "lemma": parts[6], "pos": parts[1], "parsing": parts[2], } verses[ref_key]["words"].append(word_data) # Build full text for each verse for ref_key, data in verses.items(): words = [w["word"] for w in data["words"]] data["text"] = " ".join(words) # Convert to simple format: ref -> {text, translit} result = {} for ref_key, data in verses.items(): result[ref_key] = { "text": data["text"], "words": data["words"], } return result def parse_morphhb_xml(content: str) -> dict[str, dict]: """Parse MorphHB OSIS XML format into verse dictionary. MorphHB uses OSIS XML format with elements containing word elements. """ verses: dict[str, dict] = {} try: # Parse XML root = ET.fromstring(content) except ET.ParseError: return {} # Find all verse elements for verse_elem in root.iter( "{http://www.bibletechnologies.net/2003/OSIS/namespace}verse" ): osisID = verse_elem.get("osisID", "") if not osisID: continue # Parse osisID format: Book.Chapter.Verse parts = osisID.split(".") if len(parts) < 3: continue try: chapter = int(parts[1]) verse = int(parts[2]) except ValueError: continue ref_key = f"{chapter}:{verse}" # Collect words from this verse words = [] word_data_list = [] for w_elem in verse_elem.iter( "{http://www.bibletechnologies.net/2003/OSIS/namespace}w" ): word = w_elem.text or "" word = word.strip() if not word: continue lemma = w_elem.get("lemma", "") morph = w_elem.get("morph", "") words.append(word) word_data_list.append({ "word": word, "lemma": lemma, "morph": morph, }) if words: verses[ref_key] = { "text": " ".join(words), "words": word_data_list, } return verses def parse_openbible_json(content: str) -> dict[str, str]: """Parse OpenBible JSON format into verse dictionary. OpenBible uses 3D array: bible[book][chapter][verse] But individual book files use: book[chapter][verse] """ try: data = json.loads(content) except json.JSONDecodeError: return {} verses = {} # Handle both full bible and individual book formats if isinstance(data, list): # Individual book: [chapter][verse] for chapter_idx, chapter in enumerate(data): if not isinstance(chapter, list): continue for verse_idx, verse_text in enumerate(chapter): if verse_text: ref_key = f"{chapter_idx + 1}:{verse_idx + 1}" verses[ref_key] = verse_text return verses def download_greek_nt(output_dir: Path, book: str) -> tuple[bool, int]: """Download Greek NT from MorphGNT. Returns: Tuple of (success, verse_count) """ if book not in NT_BOOKS: return False, 0 morphgnt_code = NT_BOOKS[book]["morphgnt"] url = f"{MORPHGNT_BASE}/{morphgnt_code}-morphgnt.txt" content = download_file(url) if not content: return False, 0 verses = parse_morphgnt_book(content) # Save to JSON output_file = output_dir / "greek" / f"{book}.json" output_file.parent.mkdir(parents=True, exist_ok=True) with output_file.open("w", encoding="utf-8") as f: json.dump(verses, f, ensure_ascii=False, indent=2) return True, len(verses) def download_hebrew_ot(output_dir: Path, book: str) -> tuple[bool, int]: """Download Hebrew OT from MorphHB (Westminster Leningrad Codex). Returns: Tuple of (success, verse_count) """ if book not in OT_BOOKS: return False, 0 morphhb_code = OT_BOOKS[book]["morphhb"] url = f"{MORPHHB_BASE}/{morphhb_code}.xml" content = download_file(url) if not content: return False, 0 verses = parse_morphhb_xml(content) # Save to JSON output_file = output_dir / "hebrew" / f"{book}.json" output_file.parent.mkdir(parents=True, exist_ok=True) with output_file.open("w", encoding="utf-8") as f: json.dump(verses, f, ensure_ascii=False, indent=2) return True, len(verses) def download_kjv(output_dir: Path, book: str) -> tuple[bool, int]: """Download KJV from OpenBible. Returns: Tuple of (success, verse_count) """ if book in NT_BOOKS: openbible_name = NT_BOOKS[book]["openbible"] elif book in OT_BOOKS: openbible_name = OT_BOOKS[book]["openbible"] else: return False, 0 url = f"{OPENBIBLE_BASE}/KJV/books/{openbible_name}.json" content = download_file(url) if not content: return False, 0 verses = parse_openbible_json(content) # Save to JSON output_file = output_dir / "english" / "kjv" / f"{book}.json" output_file.parent.mkdir(parents=True, exist_ok=True) with output_file.open("w", encoding="utf-8") as f: json.dump(verses, f, ensure_ascii=False, indent=2) return True, len(verses) def download_kougo(output_dir: Path, book: str) -> tuple[bool, int]: """Download Japanese Kougo-yaku from OpenBible. Returns: Tuple of (success, verse_count) """ if book in NT_BOOKS: openbible_name = NT_BOOKS[book]["openbible"] elif book in OT_BOOKS: openbible_name = OT_BOOKS[book]["openbible"] else: return False, 0 url = f"{OPENBIBLE_BASE}/KOUGO_YAKU/books/{openbible_name}.json" content = download_file(url) if not content: return False, 0 verses = parse_openbible_json(content) # Save to Japanese kougo directory output_file = output_dir / "japanese" / "kougo" / f"{book}.json" output_file.parent.mkdir(parents=True, exist_ok=True) with output_file.open("w", encoding="utf-8") as f: json.dump(verses, f, ensure_ascii=False, indent=2) return True, len(verses) def download_book_with_progress( output_dir: Path, book: str, progress: Progress, book_task: int, ) -> dict[str, tuple[bool, int]]: """Download all sources for a book with progress updates. Returns: Dictionary of source type to (success, verse_count) """ is_ot = book in OT_BOOKS results: dict[str, tuple[bool, int]] = {} # Create sub-tasks for each source type if is_ot: sources = [ ("Hebrew", lambda: download_hebrew_ot(output_dir, book)), ("KJV", lambda: download_kjv(output_dir, book)), ("Kougo", lambda: download_kougo(output_dir, book)), ] else: sources = [ ("Greek", lambda: download_greek_nt(output_dir, book)), ("KJV", lambda: download_kjv(output_dir, book)), ("Kougo", lambda: download_kougo(output_dir, book)), ] for source_name, download_func in sources: progress.update(book_task, description=f"[cyan]{book}[/] - {source_name}") success, count = download_func() results[source_name.lower()] = (success, count) progress.advance(book_task) return results def create_progress() -> Progress: """Create a rich Progress instance with custom columns.""" return Progress( SpinnerColumn(), TextColumn("[bold blue]{task.description}"), BarColumn(bar_width=40), TaskProgressColumn(), MofNCompleteColumn(), TimeElapsedColumn(), console=console, expand=False, ) def download_all_books( output_dir: Path, books: dict[str, dict], testament_name: str, ) -> dict[str, dict[str, tuple[bool, int]]]: """Download all books with visual progress bars. Args: output_dir: Output directory for source files. books: Dictionary of book names to their metadata. testament_name: "Old Testament" or "New Testament" for display. Returns: Dictionary of book name to source results. """ all_results: dict[str, dict[str, tuple[bool, int]]] = {} console.print(f"\n[bold green]{testament_name}[/] ({len(books)} books)") console.print("─" * 60) with create_progress() as progress: # Overall progress for all books overall_task = progress.add_task( f"[yellow]Downloading {testament_name}...", total=len(books), ) for book in books: # Per-book progress (3 sources each) book_task = progress.add_task( f"[cyan]{book}[/]", total=3, ) results = download_book_with_progress( output_dir, book, progress, book_task ) all_results[book] = results # Mark book task as complete and update overall progress.update(book_task, visible=False) progress.advance(overall_task) return all_results def print_summary( ot_results: dict[str, dict[str, tuple[bool, int]]] | None, nt_results: dict[str, dict[str, tuple[bool, int]]] | None, ) -> None: """Print a summary table of download results.""" console.print("\n[bold]Download Summary[/]") console.print("=" * 60) table = Table(show_header=True, header_style="bold magenta") table.add_column("Testament", style="cyan") table.add_column("Books", justify="right") table.add_column("Source", justify="right") table.add_column("KJV", justify="right") table.add_column("Kougo", justify="right") table.add_column("Total Verses", justify="right") if ot_results: ot_source_ok = sum(1 for r in ot_results.values() if r.get("hebrew", (False, 0))[0]) ot_kjv_ok = sum(1 for r in ot_results.values() if r.get("kjv", (False, 0))[0]) ot_kougo_ok = sum(1 for r in ot_results.values() if r.get("kougo", (False, 0))[0]) ot_verses = sum(r.get("hebrew", (False, 0))[1] for r in ot_results.values()) table.add_row( "Old Testament", str(len(ot_results)), f"[green]{ot_source_ok}[/] Hebrew", f"[green]{ot_kjv_ok}[/]", f"[green]{ot_kougo_ok}[/]", f"{ot_verses:,}", ) if nt_results: nt_source_ok = sum(1 for r in nt_results.values() if r.get("greek", (False, 0))[0]) nt_kjv_ok = sum(1 for r in nt_results.values() if r.get("kjv", (False, 0))[0]) nt_kougo_ok = sum(1 for r in nt_results.values() if r.get("kougo", (False, 0))[0]) nt_verses = sum(r.get("greek", (False, 0))[1] for r in nt_results.values()) table.add_row( "New Testament", str(len(nt_results)), f"[green]{nt_source_ok}[/] Greek", f"[green]{nt_kjv_ok}[/]", f"[green]{nt_kougo_ok}[/]", f"{nt_verses:,}", ) console.print(table) # Total total_books = (len(ot_results) if ot_results else 0) + (len(nt_results) if nt_results else 0) total_verses = ( (sum(r.get("hebrew", (False, 0))[1] for r in ot_results.values()) if ot_results else 0) + (sum(r.get("greek", (False, 0))[1] for r in nt_results.values()) if nt_results else 0) ) console.print(f"\n[bold green]Total: {total_books} books, {total_verses:,} verses[/]") def download_single_book(output_dir: Path, book: str) -> None: """Download a single book with progress display.""" if book in OT_BOOKS: testament = "Old Testament" books = {book: OT_BOOKS[book]} elif book in NT_BOOKS: testament = "New Testament" books = {book: NT_BOOKS[book]} else: console.print(f"[red]Unknown book: {book}[/]") return results = download_all_books(output_dir, books, testament) # Show results for single book book_results = results.get(book, {}) console.print(f"\n[bold]{book.title()}[/] download complete:") for source, (success, count) in book_results.items(): status = "[green]OK[/]" if success else "[red]FAILED[/]" console.print(f" {source}: {status} ({count} verses)") def main() -> None: """Main entry point.""" import argparse parser = argparse.ArgumentParser( description="Download Bible source texts with progress display" ) parser.add_argument( "--book", default=None, help="Single book to download", ) parser.add_argument( "--output", default="data/sources", help="Output directory (default: data/sources)", ) parser.add_argument( "--all-nt", action="store_true", help="Download all NT books", ) parser.add_argument( "--all-ot", action="store_true", help="Download all OT books", ) parser.add_argument( "--all", action="store_true", help="Download all books (OT + NT)", ) args = parser.parse_args() output_dir = Path(args.output) console.print("[bold]Bible Source Downloader[/]") console.print(f"Output directory: {output_dir}") ot_results = None nt_results = None if args.all: ot_results = download_all_books(output_dir, OT_BOOKS, "Old Testament") nt_results = download_all_books(output_dir, NT_BOOKS, "New Testament") print_summary(ot_results, nt_results) elif args.all_ot: ot_results = download_all_books(output_dir, OT_BOOKS, "Old Testament") print_summary(ot_results, None) elif args.all_nt: nt_results = download_all_books(output_dir, NT_BOOKS, "New Testament") print_summary(None, nt_results) elif args.book: download_single_book(output_dir, args.book) else: # Default: download john download_single_book(output_dir, "john") if __name__ == "__main__": main()