software/scripts/verify_output.py

1#!/usr/bin/env python32"""Read-only structural and site-reference audit for 日常訳第1版 (stdlib only)."""3from __future__ import annotations45import argparse6import csv7import hashlib8import json9import re10from collections import Counter11from pathlib import Path1213ROOT = Path(__file__).resolve().parents[3]14DEFAULT_SOURCE = ROOT / 'cli/translate-bible/data/output'15DEFAULT_BASELINE = ROOT / 'web/japanbible-net/web/public/JapKougo.csv'16TRANSLATION = '日常訳第1版'171819def book_id(name):20    name = re.sub(r'^III ', '3', name)21    name = re.sub(r'^II ', '2', name)22    name = re.sub(r'^I ', '1', name)23    return name.lower().replace(' ', '').replace('_', '').replace('revelationofjohn', 'revelation')242526def unique_object(pairs):27    result = {}28    for key, value in pairs:29        if key in result:30            raise ValueError(f'duplicate JSON key: {key}')31        result[key] = value32    return result333435def read_baseline(path):36    result = {}37    with path.open(encoding='utf-8-sig', newline='') as stream:38        reader = csv.DictReader(stream, strict=True)39        if reader.fieldnames != ['Book', 'Chapter', 'Verse', 'Text']:40            raise ValueError('baseline must have Book,Chapter,Verse,Text header')41        for row in reader:42            if None in row or any(value is None for value in row.values()):43                raise ValueError('malformed baseline row')44            key = (book_id(row['Book']), int(row['Chapter']), int(row['Verse']))45            if key in result or min(key[1:]) < 1:46                raise ValueError(f'invalid/duplicate baseline reference: {key}')47            result[key] = row['Text']48    if len({key[0] for key in result}) != 66 or len({key[:2] for key in result}) != 1189:49        raise ValueError('baseline must contain all 66 books and 1,189 chapters')50    return result515253def audit(source, baseline):54    issues, rows, manifest = [], {}, []55    expected_books = {key[0] for key in baseline}56    expected_chapters = {key[:2] for key in baseline}57    seen_chapters = set()5859    def issue(code, location, detail):60        issues.append({'code': code, 'location': location, 'detail': detail})6162    if not source.is_dir():63        issue('missing-source', str(source), 'Output directory does not exist')64    for directory in sorted(source.iterdir()) if source.is_dir() else []:65        if directory.name.startswith('.'):66            continue67        book = book_id(directory.name)68        if not directory.is_dir() or book not in expected_books:69            issue('unexpected-path', directory.name, 'Expected a canonical book directory')70            continue71        for path in sorted(directory.iterdir()):72            if path.name.startswith('.'):73                continue74            location = str(path.relative_to(source))75            match = re.fullmatch(r'chapter_([0-9]+)\.json', path.name)76            if not path.is_file() or not match:77                issue('unexpected-path', location, 'Expected chapter_NN.json')78                continue79            chapter = int(match[1])80            if path.name != f'chapter_{chapter:02d}.json':81                issue('noncanonical-filename', location, 'Filename is not chapter_NN.json')82            if (book, chapter) in seen_chapters:83                issue('duplicate-chapter', location, 'Multiple files map to the same chapter')84            seen_chapters.add((book, chapter))85            if (book, chapter) not in expected_chapters:86                issue('unexpected-chapter', location, 'Chapter absent from site baseline')87            try:88                raw = path.read_bytes()89                manifest.append({'path': location, 'sha256': hashlib.sha256(raw).hexdigest()})90                data = json.loads(raw, object_pairs_hook=unique_object)91            except (OSError, ValueError, UnicodeError) as error:92                issue('invalid-json', location, str(error))93                continue94            if not isinstance(data, dict):95                issue('invalid-schema', location, 'Chapter must be an object')96                continue97            if data.get('book') != directory.name or type(data.get('chapter')) is not int or data['chapter'] != chapter:98                issue('metadata-mismatch', location, 'book/chapter metadata must match path')99            verses = data.get('verses')100            if not isinstance(verses, dict) or not verses:101                issue('invalid-schema', location, 'verses must be a nonempty object')102                continue103            for number, record in verses.items():104                ref = f'{book} {chapter}:{number}'105                if not re.fullmatch(r'[1-9][0-9]*', number):106                    issue('invalid-verse-number', ref, 'Expected positive integer string without leading zeros')107                    continue108                key = (book, chapter, int(number))109                if key in rows:110                    issue('duplicate-verse', ref, 'Multiple records map to this reference')111                    continue112                if not isinstance(record, dict) or not isinstance(record.get('text'), str) or not record['text'].strip():113                    issue('invalid-text', ref, 'Expected nonempty text string')114                    continue115                text = record['text']116                rows[key] = text117                if re.search(r'[\r\n\x00-\x08\x0b-\x1f\x7f\u2028\u2029]', text):118                    issue('control-character', ref, 'Control/newline character incompatible with line-based site CSV parser')119                if re.search(r'<[^>]*>|\ufffd|```', text):120                    issue('invalid-plain-text', ref, 'Markup, replacement character or code fence in plain text')121                if text != text.strip():122                    issue('surrounding-whitespace', ref, 'Site parser would trim this text')123                ruby = record.get('text_with_furigana')124                if ruby is not None:125                    if not isinstance(ruby, str):126                        issue('invalid-furigana', ref, 'text_with_furigana must be a string')127                    elif ruby.strip():128                        plain = re.sub(r'<rt\b[^>]*>.*?</rt>|<rp\b[^>]*>.*?</rp>', '', ruby)129                        plain = re.sub(r'</?(?:ruby|rb)\b[^>]*>', '', plain)130                        if plain != text:131                            issue('furigana-mismatch', ref, 'Ruby base text differs from plain text or contains unsupported markup')132    for book, chapter in sorted(expected_chapters - seen_chapters):133        issue('missing-chapter', f'{book} {chapter}', 'No chapter file')134    for key in sorted(baseline.keys() - rows.keys()):135        issue('missing-verse', f'{key[0]} {key[1]}:{key[2]}', 'No valid text at site reference; review versification before publishing')136    for key in sorted(rows.keys() - baseline.keys()):137        issue('extra-verse', f'{key[0]} {key[1]}:{key[2]}', 'Reference absent from site corpus; review versification before publishing')138    summary = {139        'books': len({key[0] for key in rows}), 'chapter_files': len(manifest),140        'chapters': len({key[:2] for key in rows}), 'valid_text_records': len(rows),141        'baseline_verses': len(baseline), 'baseline_empty_texts': sum(not text.strip() for text in baseline.values()), 'matching_references': len(rows.keys() & baseline.keys()),142        'issues': dict(sorted(Counter(item['code'] for item in issues).items())),143    }144    report = {'translation': TRANSLATION, 'source': str(source.resolve()), 'summary': summary,145              'passed': not issues, 'issues': issues, 'source_manifest': manifest}146    return report, rows147148149def main():150    parser = argparse.ArgumentParser(description=__doc__)151    parser.add_argument('--source', type=Path, default=DEFAULT_SOURCE)152    parser.add_argument('--baseline', type=Path, default=DEFAULT_BASELINE)153    parser.add_argument('--report', type=Path, required=True, help='JSON report destination (outside source)')154    args = parser.parse_args()155    if args.report.resolve().is_relative_to(args.source.resolve()) or args.report.resolve() == args.baseline.resolve():156        parser.error('report must not overwrite source data or baseline')157    try:158        baseline = read_baseline(args.baseline)159        report, _ = audit(args.source, baseline)160        report['baseline'] = str(args.baseline.resolve())161        report['baseline_sha256'] = hashlib.sha256(args.baseline.read_bytes()).hexdigest()162        args.report.parent.mkdir(parents=True, exist_ok=True)163        args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')164    except (OSError, ValueError, csv.Error) as error:165        parser.exit(2, f'Cannot audit: {error}\n')166    print(TRANSLATION)167    print(json.dumps(report['summary'], ensure_ascii=False, indent=2))168    print(f'Report: {args.report}')169    return 0 if report['passed'] else 1170171172if __name__ == '__main__':173    raise SystemExit(main())174