tools/collect-provenance.py

1"""Export hash-matched historical outputs and an allowlisted input inventory.23Never export user prompts: they embed separately licensed reference translations.4System prompts are project-authored instructions, deduplicated by SHA-256.5"""6import argparse7from collections import Counter8import hashlib9import json10from pathlib import Path111213def digest(raw):14    return hashlib.sha256(raw).hexdigest()151617def collect(source, audit, destination, recovered=None):18    destination.mkdir(parents=True, exist_ok=True)19    records, systems, chapters = [], {}, []20    source_names = Counter()21    with_prompts = 022    unavailable = []23    for name, expected in sorted(audit['input_files'].items()):24        path = source / name25        origin = 'current source output'26        if (not path.is_file() or digest(path.read_bytes()) != expected) and recovered is not None:27            candidate = recovered / name28            if candidate.is_file() and digest(candidate.read_bytes()) == expected:29                path = candidate30                origin = 'recovered historical source output; exact audit hash match'31        if not path.is_file() or digest(path.read_bytes()) != expected:32            unavailable.append({'file': name, 'expected_sha256': expected, 'status': 'changed' if path.is_file() else 'missing'})33            continue34        chapter = json.loads(path.read_bytes())35        chapters.append({'file': name, 'sha256': expected, 'origin': origin})36        for verse, saved in sorted(chapter['verses'].items(), key=lambda item: int(item[0])):37            row = {'reference': f"{chapter['book']}:{chapter['chapter']}:{verse}",38                   'source_file': name, 'output_kind': 'saved parsed translation, not raw model response'}39            for field in ('text', 'text_with_furigana', 'model', 'timestamp', 'prompt_version', 'sources_used'):40                row[field] = saved.get(field)41            provenance = saved.get('provenance')42            if provenance:43                system = provenance.get('system_prompt', '')44                user = provenance.get('user_prompt', '')45                systems[digest(system.encode())] = system46                row['input'] = {'system_prompt_sha256': digest(system.encode()),47                                'user_prompt_sha256': digest(user.encode()),48                                'user_prompt_characters': len(user),49                                'user_prompt_status': 'retained privately; embeds separately licensed Bible text',50                                'source_text_sha256': {k: digest(v.encode()) for k, v in provenance.get('source_texts', {}).items()},51                                'reference_labels_in_prompt': [label for label in ('口語訳 (1955)', 'KJV', 'ASV', '明治訳', '大正訳') if label in user]}52                for field in ('started_at', 'completed_at', 'attempt_number', 'style_config_hash', 'glossary_hash'):53                    row['input'][field] = provenance.get(field)54                with_prompts += bool(system and user)55                source_names.update(row['input']['reference_labels_in_prompt'])56            else:57                row['input'] = None58            records.append(row)59    for name, value in [('outputs.jsonl', ''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in records)),60                        ('system-prompts.json', json.dumps(systems, ensure_ascii=False, indent=2)+'\n')]:61        (destination/name).write_text(value)62    report = {'schema': 1, 'edition': 'nichijo-v1', 'chapters': len(chapters), 'source_records': len(records),63              'records_with_retained_prompts': with_prompts, 'records_without_retained_prompts': len(records)-with_prompts,64              'reference_labels_in_retained_prompts': dict(source_names), 'system_prompts': len(systems),65              'limits': ['Only chapter files matching the immutable release audit are included.',66                         'Saved parsed output is not the complete raw model response.',67                         'Input records are absent for chunk translations; absence is not proof that reference text was excluded.',68                         'Historical prompts include reference translations; a no-reference-input claim is contradicted by these records.',69                         'Model and source labels are saved local metadata, not independently verified provider records.',70                         'Exact user prompts remain private because they embed separately licensed Bible text. Hashes permit later verification.'],71              'source_files': chapters, 'unavailable_files': unavailable,72              'files': {p.name: digest(p.read_bytes()) for p in (destination/'outputs.jsonl', destination/'system-prompts.json')}}73    (destination/'index.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')74    return report757677if __name__ == '__main__':78    parser=argparse.ArgumentParser()79    parser.add_argument('--source-output', type=Path, required=True)80    parser.add_argument('--audit', type=Path, required=True)81    parser.add_argument('--destination', type=Path, required=True)82    parser.add_argument('--recovered-output', type=Path)83    args=parser.parse_args()84    report=collect(args.source_output,json.loads(args.audit.read_text()),args.destination,args.recovered_output)85    print(json.dumps({k:v for k,v in report.items() if k not in ('source_files','files')},ensure_ascii=False,indent=2))86