"""Independent frozen-executor evaluation of base/adapted quantum programs.""" import ast import hashlib import json import re from collections import defaultdict from pathlib import Path import numpy as np import pyarrow.parquet as pq from huggingface_hub import HfApi, hf_hub_download import quantum_sandbox as sandbox REPO = "itsalissonsilva/QuantumCoder-9B" def main(): api = HfApi() revision = api.model_info(REPO).sha def get(path): return Path(hf_hub_download(REPO, path, revision=revision)) preflight = json.loads(get("executor_preflight/preflight.json").read_text()) assert preflight["all_passed"] and len(preflight["reference_checks"]) == 90 source_hash = hashlib.sha256(Path(sandbox.__file__).read_bytes()).hexdigest() assert ( source_hash == hashlib.sha256(get("executor_preflight/quantum_sandbox.py").read_bytes()).hexdigest() ), "Do not change the executor after opening model programs" generations = json.loads(get("hf_pilot/generation_report.json").read_text()) data = {} for split in ["validation", "test"]: rows = pq.read_table( hf_hub_download( generations["data_repo"], f"data/{split}.parquet", repo_type="dataset", revision=generations["data_revision"], ) ).to_pylist() data.update({r["id"]: r for r in rows}) ids = [r["id"] for r in generations["conditions"]["base"]] assert len(ids) == 192 and len(set(ids)) == 192 assert ids == [r["id"] for r in generations["conditions"]["adapted"]] versions = sandbox.initialize() isolation = sandbox.run("", 1, mode="isolation_check") assert isolation == preflight["isolation"] def execute(program, row): # Unwrap only a single full-output code fence. No prose, repairs or # target-informed program edits are allowed. wrapped = re.fullmatch(r"\s*```(?:python)?\s*\n(.*?)\n```\s*", program, flags=re.S) clean = wrapped.group(1) if wrapped else program syntax = True imports = set() try: tree = ast.parse(clean) for node in ast.walk(tree): if isinstance(node, ast.Import): imports.update(a.name.split(".")[0] for a in node.names) elif isinstance(node, ast.ImportFrom): imports.add((node.module or "").split(".")[0]) except SyntaxError: syntax = False result = sandbox.run(clean, row["qubits"]) fidelity = None state_pass = False if result.get("status") == "executed": actual = np.array(result["real"]) + 1j * np.array(result["imag"]) target = np.array(row["state_real"]) + 1j * np.array(row["state_imag"]) fidelity = float(abs(np.vdot(target, actual)) ** 2) state_pass = abs(result["norm"] - 1) < 1e-6 and fidelity > 1 - 1e-6 requested = row["framework"] in imports execution = {k: v for k, v in result.items() if k not in ["real", "imag"]} return dict( syntax_parses=syntax, requested_framework_import=requested, single_fence_unwrapped=wrapped is not None, execution=execution, state_pass=state_pass, fidelity=fidelity, passed=state_pass and requested, program_sha256=hashlib.sha256(program.encode()).hexdigest(), ) # Every selected reference must pass the same evaluator before scoring models. reference_checks = [] for identifier in ids: row = data[identifier] outcome = execute(row["program"], row) assert outcome["passed"], (identifier, outcome) reference_checks.append(dict(id=identifier, **outcome)) records = [] for condition, outputs in generations["conditions"].items(): for output in outputs: row = data[output["id"]] assert row["framework"] == output["framework"] and row["qubits"] == output["qubits"] assert np.allclose(row["state_real"], output["target_real"], atol=1e-12, rtol=0) assert np.allclose(row["state_imag"], output["target_imag"], atol=1e-12, rtol=0) records.append( dict( condition=condition, id=row["id"], group_id=row["group_id"], split=output["split"], framework=row["framework"], generated_tokens=output["generated_tokens"], possible_token_cap=output["generated_tokens"] >= 512, **execute(output["program"], row), ) ) print( json.dumps( dict( stage="scored", condition=condition, passed=sum(r["passed"] for r in records if r["condition"] == condition), ) ), flush=True, ) summary = {} for condition in ["base", "adapted"]: summary[condition] = {} for split in ["validation", "test"]: chosen = [r for r in records if r["condition"] == condition and r["split"] == split] summary[condition][split] = dict( n=len(chosen), passed=sum(r["passed"] for r in chosen), pass_at_1=float(np.mean([r["passed"] for r in chosen])), state_correct=sum(r["state_pass"] for r in chosen), executed=sum(r["execution"].get("status") == "executed" for r in chosen), syntax_valid=sum(r["syntax_parses"] for r in chosen), requested_import=sum(r["requested_framework_import"] for r in chosen), possible_token_cap=sum(r["possible_token_cap"] for r in chosen), ) base = {r["id"]: r for r in records if r["condition"] == "base" and r["split"] == "test"} adapted = {r["id"]: r for r in records if r["condition"] == "adapted" and r["split"] == "test"} keys = sorted(base) delta = np.array([int(adapted[k]["passed"]) - int(base[k]["passed"]) for k in keys]) groups = defaultdict(list) for i, k in enumerate(keys): groups[base[k]["group_id"]].append(i) indices = list(groups.values()) rng = np.random.default_rng(20261031) boot = [ float(delta[np.concatenate([indices[i] for i in draw])].mean()) for draw in rng.integers(0, len(indices), size=(2000, len(indices))) ] report = dict( source_revision=revision, executor_sha256=source_hash, versions=versions, isolation=isolation, data_revision=generations["data_revision"], reference_checks=reference_checks, records=records, summary=summary, paired_test=dict( improved=int(np.sum(delta == 1)), regressed=int(np.sum(delta == -1)), mean_pass_delta=float(delta.mean()), group_bootstrap_95=np.quantile(boot, [0.025, 0.975]).tolist(), groups=len(indices), ), limitations="One synthetic held-out-framework experiment. Pass requires requested-framework import, supported-Python execution, unit norm and global-phase-invariant exact-state fidelity >1-1e-6. Import presence is a proxy for framework compliance, not a formal data-flow proof. Unsupported valid Python is rejected and can yield conservative false negatives. Only a full-output code fence may be removed. No repairs, retries or generated-code access to files/network. Original batch-width at_token_cap flag is unreliable and not used; generated_tokens>=512 is reported as a possible cap, not an exact stop reason.", ) report["release_gate_pass"] = ( summary["adapted"]["test"]["passed"] > summary["base"]["test"]["passed"] ) folder = Path("quantum_execution") folder.mkdir(exist_ok=True) (folder / "execution.json").write_text(json.dumps(report, indent=2)) (folder / "train_quantum_execute_pilot.py").write_bytes(Path(__file__).read_bytes()) (folder / "quantum_sandbox.py").write_bytes(Path(sandbox.__file__).read_bytes()) api.upload_folder( repo_id=REPO, folder_path=str(folder), path_in_repo="execution", commit_message="Evaluate frozen base/adapter programs under audited nobody/seccomp runner with matched reference checks and exact quantum-state fidelity", ) print( json.dumps( dict(summary=summary, gate=report["release_gate_pass"], paired=report["paired_test"]) ), flush=True, ) if __name__ == "__main__": main()