software/scripts/review_output.py

1#!/usr/bin/env python32"""Produce a heuristic human-review queue; flags are not translation judgments."""3from __future__ import annotations45import argparse6import csv7import re8from collections import Counter, defaultdict9from pathlib import Path1011from verify_output import DEFAULT_BASELINE, DEFAULT_SOURCE, TRANSLATION, audit, read_baseline121314def review(rows, baseline):15    duplicates = defaultdict(list)16    for key, text in rows.items():17        duplicates[(key[0], key[1], text.strip())].append(key[2])18    for key, text in sorted(rows.items()):19        flags = []20        original = baseline.get(key, '')21        if not re.search(r'[\u3040-\u30ff\u3400-\u9fff]', text):22            flags.append('no-japanese')23        if re.search(r'[\u0370-\u03ff\u0590-\u05ff\u1f00-\u1fff]', text):24            flags.append('source-script')25        if re.search(r'[A-Za-z]{2,}', text):26            flags.append('latin-text')27        if re.search(r'TODO|TBD|未翻訳|翻訳できません|翻訳は以下|翻訳結果|訳文：', text, re.I):28            flags.append('placeholder-or-commentary')29        if re.search(r'(.{8,}?)\1{2,}', text):30            flags.append('repeated-phrase')31        if len(duplicates[(key[0], key[1], text.strip())]) > 1:32            flags.append('identical-within-chapter')33        if len(text) < 5:34            flags.append('very-short')35        if original and len(original) >= 30 and len(text) < len(original) * 0.35:36            flags.append('short-vs-kougo')37        if original and len(text) > max(100, len(original) * 3):38            flags.append('long-vs-kougo')39        if flags:40            yield {'book': key[0], 'chapter': key[1], 'verse': key[2],41                   'flags': ';'.join(flags), 'text': text, 'kougo_text': original}424344def main():45    parser = argparse.ArgumentParser(description=__doc__)46    parser.add_argument('--source', type=Path, default=DEFAULT_SOURCE)47    parser.add_argument('--baseline', type=Path, default=DEFAULT_BASELINE)48    parser.add_argument('--report', type=Path, required=True, help='Review CSV destination (outside source)')49    args = parser.parse_args()50    if args.report.resolve().is_relative_to(args.source.resolve()) or args.report.resolve() == args.baseline.resolve():51        parser.error('report must not overwrite source data or baseline')52    try:53        baseline = read_baseline(args.baseline)54        structural, rows = audit(args.source, baseline)55        queue = list(review(rows, baseline))56        args.report.parent.mkdir(parents=True, exist_ok=True)57        with args.report.open('w', encoding='utf-8', newline='') as stream:58            writer = csv.DictWriter(stream, fieldnames=['book', 'chapter', 'verse', 'flags', 'text', 'kougo_text'])59            writer.writeheader()60            writer.writerows(queue)61    except (OSError, ValueError, csv.Error) as error:62        parser.exit(2, f'Cannot review: {error}\n')63    print(f'{TRANSLATION}: {len(queue)} verses flagged for human review')64    for flag, count in sorted(Counter(flag for row in queue for flag in row['flags'].split(';')).items()):65        print(f'  {flag}: {count}')66    print(f'Structural issues: {len(structural["issues"])} (run verify_output.py for details)')67    print(f'Report: {args.report}')68    return 0 if structural['passed'] else 1697071if __name__ == '__main__':72    raise SystemExit(main())73