#!/usr/bin/env python3 """Read-only structural and site-reference audit for 日常訳第1版 (stdlib only).""" from __future__ import annotations import argparse import csv import hashlib import json import re from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[3] DEFAULT_SOURCE = ROOT / 'cli/translate-bible/data/output' DEFAULT_BASELINE = ROOT / 'web/japanbible-net/web/public/JapKougo.csv' TRANSLATION = '日常訳第1版' def book_id(name): name = re.sub(r'^III ', '3', name) name = re.sub(r'^II ', '2', name) name = re.sub(r'^I ', '1', name) return name.lower().replace(' ', '').replace('_', '').replace('revelationofjohn', 'revelation') def unique_object(pairs): result = {} for key, value in pairs: if key in result: raise ValueError(f'duplicate JSON key: {key}') result[key] = value return result def read_baseline(path): result = {} with path.open(encoding='utf-8-sig', newline='') as stream: reader = csv.DictReader(stream, strict=True) if reader.fieldnames != ['Book', 'Chapter', 'Verse', 'Text']: raise ValueError('baseline must have Book,Chapter,Verse,Text header') for row in reader: if None in row or any(value is None for value in row.values()): raise ValueError('malformed baseline row') key = (book_id(row['Book']), int(row['Chapter']), int(row['Verse'])) if key in result or min(key[1:]) < 1: raise ValueError(f'invalid/duplicate baseline reference: {key}') result[key] = row['Text'] if len({key[0] for key in result}) != 66 or len({key[:2] for key in result}) != 1189: raise ValueError('baseline must contain all 66 books and 1,189 chapters') return result def audit(source, baseline): issues, rows, manifest = [], {}, [] expected_books = {key[0] for key in baseline} expected_chapters = {key[:2] for key in baseline} seen_chapters = set() def issue(code, location, detail): issues.append({'code': code, 'location': location, 'detail': detail}) if not source.is_dir(): issue('missing-source', str(source), 'Output directory does not exist') for directory in sorted(source.iterdir()) if source.is_dir() else []: if directory.name.startswith('.'): continue book = book_id(directory.name) if not directory.is_dir() or book not in expected_books: issue('unexpected-path', directory.name, 'Expected a canonical book directory') continue for path in sorted(directory.iterdir()): if path.name.startswith('.'): continue location = str(path.relative_to(source)) match = re.fullmatch(r'chapter_([0-9]+)\.json', path.name) if not path.is_file() or not match: issue('unexpected-path', location, 'Expected chapter_NN.json') continue chapter = int(match[1]) if path.name != f'chapter_{chapter:02d}.json': issue('noncanonical-filename', location, 'Filename is not chapter_NN.json') if (book, chapter) in seen_chapters: issue('duplicate-chapter', location, 'Multiple files map to the same chapter') seen_chapters.add((book, chapter)) if (book, chapter) not in expected_chapters: issue('unexpected-chapter', location, 'Chapter absent from site baseline') try: raw = path.read_bytes() manifest.append({'path': location, 'sha256': hashlib.sha256(raw).hexdigest()}) data = json.loads(raw, object_pairs_hook=unique_object) except (OSError, ValueError, UnicodeError) as error: issue('invalid-json', location, str(error)) continue if not isinstance(data, dict): issue('invalid-schema', location, 'Chapter must be an object') continue if data.get('book') != directory.name or type(data.get('chapter')) is not int or data['chapter'] != chapter: issue('metadata-mismatch', location, 'book/chapter metadata must match path') verses = data.get('verses') if not isinstance(verses, dict) or not verses: issue('invalid-schema', location, 'verses must be a nonempty object') continue for number, record in verses.items(): ref = f'{book} {chapter}:{number}' if not re.fullmatch(r'[1-9][0-9]*', number): issue('invalid-verse-number', ref, 'Expected positive integer string without leading zeros') continue key = (book, chapter, int(number)) if key in rows: issue('duplicate-verse', ref, 'Multiple records map to this reference') continue if not isinstance(record, dict) or not isinstance(record.get('text'), str) or not record['text'].strip(): issue('invalid-text', ref, 'Expected nonempty text string') continue text = record['text'] rows[key] = text if re.search(r'[\r\n\x00-\x08\x0b-\x1f\x7f\u2028\u2029]', text): issue('control-character', ref, 'Control/newline character incompatible with line-based site CSV parser') if re.search(r'<[^>]*>|\ufffd|```', text): issue('invalid-plain-text', ref, 'Markup, replacement character or code fence in plain text') if text != text.strip(): issue('surrounding-whitespace', ref, 'Site parser would trim this text') ruby = record.get('text_with_furigana') if ruby is not None: if not isinstance(ruby, str): issue('invalid-furigana', ref, 'text_with_furigana must be a string') elif ruby.strip(): plain = re.sub(r']*>.*?|]*>.*?', '', ruby) plain = re.sub(r']*>', '', plain) if plain != text: issue('furigana-mismatch', ref, 'Ruby base text differs from plain text or contains unsupported markup') for book, chapter in sorted(expected_chapters - seen_chapters): issue('missing-chapter', f'{book} {chapter}', 'No chapter file') for key in sorted(baseline.keys() - rows.keys()): issue('missing-verse', f'{key[0]} {key[1]}:{key[2]}', 'No valid text at site reference; review versification before publishing') for key in sorted(rows.keys() - baseline.keys()): issue('extra-verse', f'{key[0]} {key[1]}:{key[2]}', 'Reference absent from site corpus; review versification before publishing') summary = { 'books': len({key[0] for key in rows}), 'chapter_files': len(manifest), 'chapters': len({key[:2] for key in rows}), 'valid_text_records': len(rows), 'baseline_verses': len(baseline), 'baseline_empty_texts': sum(not text.strip() for text in baseline.values()), 'matching_references': len(rows.keys() & baseline.keys()), 'issues': dict(sorted(Counter(item['code'] for item in issues).items())), } report = {'translation': TRANSLATION, 'source': str(source.resolve()), 'summary': summary, 'passed': not issues, 'issues': issues, 'source_manifest': manifest} return report, rows def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=Path, default=DEFAULT_SOURCE) parser.add_argument('--baseline', type=Path, default=DEFAULT_BASELINE) parser.add_argument('--report', type=Path, required=True, help='JSON report destination (outside source)') args = parser.parse_args() if args.report.resolve().is_relative_to(args.source.resolve()) or args.report.resolve() == args.baseline.resolve(): parser.error('report must not overwrite source data or baseline') try: baseline = read_baseline(args.baseline) report, _ = audit(args.source, baseline) report['baseline'] = str(args.baseline.resolve()) report['baseline_sha256'] = hashlib.sha256(args.baseline.read_bytes()).hexdigest() args.report.parent.mkdir(parents=True, exist_ok=True) args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') except (OSError, ValueError, csv.Error) as error: parser.exit(2, f'Cannot audit: {error}\n') print(TRANSLATION) print(json.dumps(report['summary'], ensure_ascii=False, indent=2)) print(f'Report: {args.report}') return 0 if report['passed'] else 1 if __name__ == '__main__': raise SystemExit(main())