#!/usr/bin/env python3
"""Version 1.0, 2026-09-09. Score authored rule-conformance teaching cases.

Runs a local, inspected copy of KevinBytes identity_hunt.py. No downloads,
AI calls, credentials, cloud operations or persistent database are used.
"""
import contextlib
import io
import json
from pathlib import Path
import runpy
import sys


def score(labels, predictions):
    if set(labels) != set(predictions):
        raise ValueError('Prediction cases must exactly match label cases')
    counts = dict(TP=0, FP=0, FN=0, TN=0)
    for name, label in labels.items():
        predicted = predictions[name]
        if type(label) is not int or label not in (0, 1):
            raise ValueError('Labels must be binary integers')
        if type(predicted) is not int or predicted not in (0, 1):
            raise ValueError('Predictions must be binary integers')
        counts[('TP' if label else 'FP') if predicted else ('FN' if label else 'TN')] += 1
    tp, fp, fn = counts['TP'], counts['FP'], counts['FN']
    return dict(counts, total=sum(counts.values()),
                precision=tp / (tp + fp) if tp + fp else None,
                recall=tp / (tp + fn) if tp + fn else None)


def main():
    folder = Path(__file__).resolve().parent
    source = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else folder / 'identity_hunt.py'
    if not source.is_file():
        raise SystemExit('Download and inspect identity_hunt.py first; see README.md.')
    fixture = json.loads((folder / 'detection-fixture.json').read_text())
    captured = io.StringIO()
    sys.dont_write_bytecode = True
    with contextlib.redirect_stdout(captured):
        runpy.run_path(str(source), run_name='__main__')
    baseline = json.loads(captured.getvalue())
    cases = fixture['cases']
    assert set(baseline['cases']) == set(cases), 'Original case set changed'
    assert baseline['fixture_rows'] == 35, 'Original row count changed'
    assert len(baseline['combined_matches']) == 2, 'Original combined results changed'
    labels = {name: case['label'] for name, case in cases.items()}
    for name, observed in baseline['cases'].items():
        assert observed['expected_matches'] == labels[name], (name, 'Label mismatch')
        assert observed['observed_matches'] == labels[name], (name, 'Baseline mismatch')
    predictions = {
        'baseline': {name: int(case['observed_matches'] > 0) for name, case in baseline['cases'].items()},
        'simulated_suggestion': {name: case['suggestion'] for name, case in cases.items()},
        'example_analyst': {name: case['analyst'] for name, case in cases.items()},
    }
    results = {stage: score(labels, values) for stage, values in predictions.items()}
    expected = {'baseline': (2, 0, 0, 6), 'simulated_suggestion': (1, 2, 1, 4),
                'example_analyst': (2, 0, 0, 6)}
    for stage, counts in results.items():
        assert tuple(counts[key] for key in ('TP', 'FP', 'FN', 'TN')) == expected[stage]
        assert counts['total'] == 8
    print(json.dumps({'unit': 'one named fixture case', 'fixture_rows': baseline['fixture_rows'],
                      'scores': results,
                      'telemetry_gap': cases['missing_success_telemetry']['operational_disposition'],
                      'limits': 'Observable-rule conformance only. No attack labels, AI model, live detection or participant performance measured.'}, indent=2))


if __name__ == '__main__':
    main()