tools/package-source.py

1"""Prepare a public source distribution from an explicit, inspected allowlist.23Never copy the monorepo's .git, .env files, credentials or private run archives.4This deliberately leaves gaps in historical input/output rather than publishing5unrelated private conversations or pretending they have been recovered.6"""7import argparse8import hashlib9import json10from pathlib import Path11import shutil12import subprocess1314SITE=Path(__file__).resolve().parents[1]15ROOT=SITE.parents[1]161718def package(out):19    if out.exists() and any(out.iterdir()):20        raise ValueError('Use an empty destination')21    out.mkdir(parents=True, exist_ok=True)22    corpus=SITE/'public/downloads/nichijo-v1'23    shutil.copytree(corpus,out/'edition-1')24    shutil.copytree(SITE/'public/evidence',out/'evidence')25    shutil.copytree(SITE/'public/downloads/provenance-v1',out/'provenance-v1')26    translator=ROOT/'cli/translate-bible'27    for name in ['main.py','pyproject.toml','uv.lock','README.md','LICENSE']:28        dest=out/'software'/name;dest.parent.mkdir(exist_ok=True);shutil.copyfile(translator/name,dest)29    # Python source and package data only. No caches, .env or local databases.30    for p in sorted((translator/'src').rglob('*')):31        if p.is_file() and p.suffix in ('.py','.json','.yaml','.yml') and '__pycache__' not in p.parts:32            dest=out/'software'/p.relative_to(translator);dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest)33    for folder in ['scripts', 'tests']:34        for source in sorted((translator/folder).rglob('*.py')):35            dest=out/'software'/source.relative_to(translator);dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest)36    for source in sorted((translator/'data/config').glob('*.json')):37        dest=out/'software'/source.relative_to(translator);dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(source,dest)38    for name in ['export-release.py','package-source.py','package-site-source.py','collect-provenance.py']:39        dest=out/'tools'/name;dest.parent.mkdir(exist_ok=True);shutil.copyfile(SITE/'scripts'/name,dest)40    audit_path=translator/'reports/nichijo-v1-numbering-alignment.json'41    audit=json.loads(audit_path.read_text())42    # Only translation decisions, Japanese alternatives and reference routing.43    decisions={44        'source_sha256':hashlib.sha256(audit_path.read_bytes()).hexdigest(),45        'aligned_changes':{key:{'text':v['text'],'sources':v['sources']} for key,v in audit['aligned_changes'].items()},46        'resolutions':{key:{'reason':v['reason'],'selected':v.get('selected',[]),47                           'all_candidates':[{'source':c['source'],'text':c['text']} for c in v.get('all_candidates',[])]}48                       for key,v in audit['resolutions'].items()}49    }50    (out/'evidence/numbering-decisions.json').write_text(json.dumps(decisions,ensure_ascii=False,indent=2)+'\n')51    (out/'LICENSE').write_text((corpus/'LICENSE.txt').read_text())52    (out/'.gitignore').write_text('.env\n.env.*\n.venv/\n__pycache__/\nsoftware/data/corpus/\nsoftware/data/output/\nsoftware/data/jobs/\nsoftware/data/request-archive/\n*.log\n')53    (out/'README.md').write_text('''# 日常訳 / Nichijoyaku5455An open Japanese Bible translation, maintained and stewarded by **Conor Livingston**.5657The Japanese text is **CC BY 4.0**; project software is **MIT**. See LICENSE58and software/LICENSE. The Bible was generated with AI and has not received59comprehensive human expert review. File verification does not prove accuracy.6061## Read, reuse, contribute6263- [Read edition 1](https://japanbible.net/?translation=nichijo-v1).64- [Project, method and evidence](https://nichijoyaku.pages.dev/).65- `edition-1/`: verified book JSON, JSONL, CSV, text, USFM, furigana and66  printable academic study layout. The study layout uses the same edition.67- `edition-1/checksums.json`: SHA-256 for every distribution file.68- Download the complete source archive from the project website; no GitHub69  account is needed to inspect, verify or reuse it.70- [Suggest a correction without an account](https://nichijoyaku.pages.dev/contribute/).71  Identify edition, passage, current wording or reading, proposed change and reason.72  Do not include private data.7374## What can be inspected7576`software/` contains the translation pipeline's source, not its private runtime77environment. `evidence/` includes pinned prompts/configuration, release evidence,78usage summaries, and surviving numbering decisions with Japanese alternatives.79`provenance-v1/` publishes 28,602 saved parsed outputs from 1,096 release-matched80chapter files. 6,446 include retained-input hashes, reference labels and timing;81one project-authored system prompt is available in full. Historical prompts82include Kougo and KJV, so a claim of no reference inputs is contradicted.83Exact user prompts remain private because they embed separately licensed text.8494 missing/changed chapter inputs are explicitly listed. Two Joel chapters85were recovered from Git history and verified against the original audit.86Original-language corpora and reference Bible translations are not redistributed.87Download prerequisites using the software's documented source importers.8889The prompt builder supports reference translations. We do **not** claim that90no other translations were input, that all model training data was copyright91free, or that every historical LLM input/output survives. Existing evidence is92published as evidence; missing records are not reconstructed and relabeled as93original logs. Model labels in saved records are not independent vendor checks.9495## Reproduce and verify9697From the extracted source directory: `python3 tools/verify-package.py` checks every distributed98file against its checksum manifest. No network or model calls are required.99The release-generation tools in `tools/` document the export from the source100monorepo; they need that original layout and are not standalone translators.101Use `software/README.md` for pipeline requirements. Private runtime state is not102included and running a translation may incur provider charges.103104## Changes and stewardship105106Submissions do not rewrite a published edition automatically. A change records107the old text, proposed text, reference, reason, reviewer role and decision.108Keep edition 1 immutable; publish accepted translation changes as a new edition109with its own manifest. AI review and human review must remain distinguishable.110111Why open source? Reuse should not depend on one website, device or maintainer.112People should be able to inspect the work, correct mistakes and build new113reading, teaching and accessibility tools around it.114''')115    (out/'CONTRIBUTING.md').write_text('''# Contributing116117Report a word, verse or furigana issue with the edition, book/chapter/verse,118exact current wording/reading, proposed replacement, and the reason. An account119is not needed: use https://nichijoyaku.pages.dev/contribute/ on the project website.120Do not paste private correspondence or long copyrighted commentary excerpts.121122Translation changes need review. Preserve before/after text and supporting123references. Mark AI-generated suggestions clearly; never label them human review.124Contributions to text use CC BY 4.0; software changes use MIT.125''')126    (out/'tools/verify-package.py').write_text('''import hashlib, json127from pathlib import Path128root=Path(__file__).resolve().parents[1]/'edition-1'129package=json.loads((root.parent/'PROVENANCE.json').read_text())130for name, expected in package['files'].items():131    if hashlib.sha256((root.parent/name).read_bytes()).hexdigest()!=expected:132        raise SystemExit('Source hash mismatch: '+name)133manifest=json.loads((root/'checksums.json').read_text())134for name, expected in manifest['files'].items():135    actual=hashlib.sha256((root/name).read_bytes()).hexdigest()136    if actual!=expected: raise SystemExit('Hash mismatch: '+name)137provenance=root.parent/'provenance-v1'138record=json.loads((provenance/'index.json').read_text())139for name, expected in record['files'].items():140    if hashlib.sha256((provenance/name).read_bytes()).hexdigest()!=expected:141        raise SystemExit('Provenance hash mismatch: '+name)142print('Verified',len(package['files']),'source files,',len(manifest['files']),'edition files and historical provenance')143''')144    # Record exactly which source revision was inspected without leaking paths.145    revision=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip()146    (out/'PROVENANCE.json').write_text(json.dumps({'source_repository':'jtcm-io/monorepo','base_revision':revision,147       'packaged_at':'2026-10-05','edition':'nichijo-v1','source_scope':'Explicit source allowlist; local additions for the October 5 open release.',148       'missing':'Complete historical LLM inputs/outputs, private session bodies, unpreserved correction reports.',149       'files':{p.relative_to(out).as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(out.rglob('*')) if p.is_file()}},indent=2)+'\n')150    print(f'Prepared {out}')151152if __name__=='__main__':153    parser=argparse.ArgumentParser();parser.add_argument('destination',type=Path)154    package(parser.parse_args().destination)155