SemIf-OpenJev/benchmarks/build_typesafe.py

"""Rebuild the frozen 102-row TypeSafe subset from verified public case payloads."""

from __future__ import annotations

import argparse
import hashlib
import json
import math
from pathlib import Path


def parse_payload(path: Path):
    text = path.read_text().strip()
    prefix = "__VIEWER_DATA__("
    if not text.startswith(prefix):
        raise ValueError(f"Unexpected wrapper in {path}")
    payload, end = json.JSONDecoder().raw_decode(text[len(prefix) :])
    if text[len(prefix) + end :].strip() not in {")", ");"}:
        raise ValueError(f"Unexpected trailing content in {path}")
    return payload


def vector(raw, keys):
    values = [float(raw.get(key, 0)) for key in keys]
    if any(not math.isfinite(value) or value < 0 for value in values) or sum(values) <= 0:
        raise ValueError("Invalid probability vector")
    total = sum(values)
    return [value / total for value in values]


def reference_vector(answer, keys):
    if answer.get("probabilities"):
        return vector(answer["probabilities"], keys)
    value = answer.get("value")
    key = str(value).lower() if isinstance(value, bool) else str(value)
    if key not in keys:
        raise ValueError("Reference value is outside the declared options")
    return [float(candidate == key) for candidate in keys]


def published_vector(answer, primitive, keys):
    if primitive == "noul":
        probability = float(answer["noul"])
        return [probability if key == "true" else 1 - probability for key in keys]
    return vector(answer["probabilities"], keys)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source-dir", type=Path, required=True)
    parser.add_argument("--selection", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()
    if args.output.exists():
        parser.error("Output must be new")
    manifest = [json.loads(line) for line in args.selection.read_text().splitlines() if line.strip()]
    selected = [row for row in manifest if row["source"] == "typesafe"]
    payloads = {}
    for workflow in {row["upstream"]["workflow"] for row in selected}:
        path = args.source_dir / f"typesafe-{workflow}-cases.js"
        payloads[workflow] = parse_payload(path)
    rows = []
    for item in selected:
        upstream = item["upstream"]
        workflow = upstream["workflow"]
        payload = payloads[workflow]
        # The frozen manifest hashes the human-readable parsed snapshot.
        parsed_hash = hashlib.sha256(json.dumps(payload, indent=2).encode()).hexdigest()
        if parsed_hash != upstream["snapshot_sha256"]:
            raise ValueError(f"Parsed {workflow} snapshot hash changed")
        evaluation = payload["eval"]
        case = evaluation["cases"][upstream["case_id"]]
        question = evaluation["questions"][upstream["question_index"]]
        document = evaluation["documents"][upstream["document_index"]]
        primitive = upstream["primitive"]
        criteria = question.get("criteria")
        if primitive == "noul":
            options = [
                {"id": key, "description": (criteria or {}).get(key, f"The proposition is {key}.")}
                for key in ("true", "false")
            ]
        elif primitive == "choice":
            options = [{"id": key, "description": value} for key, value in criteria.items()]
        else:
            raise ValueError("The frozen comparison excludes Score primitives")
        for option in options:
            option["description"] = option["id"] + ": " + option["description"]
        keys = [option["id"] for option in options]
        if keys != item["option_ids"]:
            raise ValueError(f"Option order changed for {item['id']}")
        references = case["reference_answers"][upstream["node"]][upstream["question_id"]]["sets"]
        distributions = [reference_vector(answer, keys) for answer in references]
        target = [sum(row[index] for row in distributions) / len(distributions) for index in range(len(keys))]
        maxima = [index for index, value in enumerate(target) if abs(value - max(target)) < 1e-12]
        if len(maxima) != 1:
            raise ValueError("Frozen TypeSafe rows require a unique reference argmax")
        published = {}
        for model_key, model_record in case["models"].items():
            for node in model_record["nodes"]:
                if (
                    node["node"] == upstream["node"]
                    and node.get("doc") == upstream["document_index"]
                    and node.get("questions", {}).get(upstream["question_id"]) == upstream["question_index"]
                    and upstream["question_id"] in node.get("answers", {})
                ):
                    answer = node["answers"][upstream["question_id"]]
                    published[model_key] = {
                        "model": model_record["model"],
                        "distribution": published_vector(answer, primitive, keys),
                    }
        if "typesafe" not in published:
            raise ValueError("Selected row lacks a published TypeSafe answer")
        rows.append(
            {
                "id": item["id"],
                "group_id": item["group_id"],
                "family": item["family"],
                "split": "external_typesafe_selected",
                "state": json.dumps(document, ensure_ascii=False, indent=2),
                "question": question["instructions"],
                "options": options,
                "label": maxima[0],
                "target_distribution": target,
                "primitive": primitive,
                "published_models": published,
                "provenance": upstream,
            }
        )
    if len(rows) != 102 or len({row["id"] for row in rows}) != 102:
        raise ValueError("Selection must rebuild exactly 102 unique TypeSafe rows")
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows))
    print("wrote 102 rows")


if __name__ == "__main__":
    main()