tools/package-site-source.py

1"""Build the website's deterministic, allowlisted source ZIP and text browser.23Run from a complete monorepo checkout. The source archive deliberately excludes4private runtime archives and separately licensed reference corpora.5"""6import argparse7import hashlib8import importlib.util9import json10from pathlib import Path11import shutil12import tempfile13import zipfile1415SITE = Path(__file__).resolve().parents[1]16LIMIT = 25 * 1024 * 102417ARCHIVE = 'nichijoyaku-source.zip'18PREFIX = 'nichijoyaku-source'19BROWSER_SUFFIXES = {'.py': 'python', '.json': 'json', '.yaml': 'yaml', '.yml': 'yaml',20                    '.toml': 'toml', '.md': 'markdown'}212223def digest(data):24    return hashlib.sha256(data).hexdigest()252627def browser_language(name):28    path = Path(name)29    if name in ('README.md', 'CONTRIBUTING.md', 'LICENSE'):30        return 'markdown' if path.suffix == '.md' else 'text'31    if path.parts[0] not in ('software', 'tools'):32        return None33    if path.name == 'LICENSE':34        return 'text'35    return BROWSER_SUFFIXES.get(path.suffix)363738def archive(source, destination):39    """Freeze an already allowlisted directory; no unreviewed tree is read here."""40    destination.mkdir(parents=True, exist_ok=True)41    code = destination / 'code'42    if code.exists():43        shutil.rmtree(code)44    files = {}45    browser = []46    with zipfile.ZipFile(destination / ARCHIVE, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=9) as bundle:47        for path in sorted(source.rglob('*')):48            if path.is_symlink():49                raise ValueError('Source package cannot contain symlinks')50            if not path.is_file():51                continue52            name = path.relative_to(source).as_posix()53            data = path.read_bytes()54            files[name] = {'sha256': digest(data), 'bytes': len(data)}55            info = zipfile.ZipInfo(f'{PREFIX}/{name}', date_time=(2026, 10, 5, 0, 0, 0))56            info.create_system = 357            info.external_attr = 0o100644 << 1658            info.compress_type = zipfile.ZIP_DEFLATED59            bundle.writestr(info, data, compresslevel=9)60            language = browser_language(name)61            if language:62                data.decode('utf-8')  # Fail rather than silently corrupt binary text.63                if len(data) >= LIMIT:64                    raise ValueError('Browser source exceeds hosting file limit')65                raw = code / (name + '.txt')66                raw.parent.mkdir(parents=True, exist_ok=True)67                raw.write_bytes(data)68                browser.append({'file': name, 'url': '/downloads/source-v1/code/' + name + '.txt',69                                'language': language, **files[name]})70    data = (destination / ARCHIVE).read_bytes()71    if len(data) >= LIMIT:72        raise ValueError('Source archive exceeds hosting file limit')73    record = {'schema_version': 1, 'archive': {'file': ARCHIVE,74              'url': '/downloads/source-v1/' + ARCHIVE, 'sha256': digest(data),75              'bytes': len(data), 'file_count': len(files)}, 'files': files,76              'browser_files': browser,77              'scope': 'Reviewed source allowlist, edition 1, and retained public provenance; no private prompts or reference Bible corpora.'}78    (destination / 'index.json').write_text(json.dumps(record, ensure_ascii=False, indent=2) + '\n')79    return record808182def build(destination):83    spec = importlib.util.spec_from_file_location('source_package', SITE / 'scripts/package-source.py')84    package = importlib.util.module_from_spec(spec)85    spec.loader.exec_module(package)86    with tempfile.TemporaryDirectory() as temporary:87        source = Path(temporary) / 'source'88        package.package(source)89        return archive(source, destination)909192if __name__ == '__main__':93    parser = argparse.ArgumentParser(description=__doc__)94    parser.add_argument('--destination', type=Path, default=SITE / 'public/downloads/source-v1')95    record = build(parser.parse_args().destination)96    print(json.dumps(record['archive']))97    print('Browsable files:', len(record['browser_files']))98