QuantumCoder-9B / execution /train_quantum_execute_pilot.py
itsalissonsilva's picture
Evaluate frozen base/adapter programs under audited nobody/seccomp runner with matched reference checks and exact quantum-state fidelity
8eb2152 verified
Raw History Blame Contribute Delete
8.67 kB
"""Independent frozen-executor evaluation of base/adapted quantum programs."""
import ast
import hashlib
import json
import re
from collections import defaultdict
from pathlib import Path
import numpy as np
import pyarrow.parquet as pq
from huggingface_hub import HfApi, hf_hub_download
import quantum_sandbox as sandbox
REPO = "itsalissonsilva/QuantumCoder-9B"
def main():
api = HfApi()
revision = api.model_info(REPO).sha
def get(path):
return Path(hf_hub_download(REPO, path, revision=revision))
preflight = json.loads(get("executor_preflight/preflight.json").read_text())
assert preflight["all_passed"] and len(preflight["reference_checks"]) == 90
source_hash = hashlib.sha256(Path(sandbox.__file__).read_bytes()).hexdigest()
assert (
source_hash
== hashlib.sha256(get("executor_preflight/quantum_sandbox.py").read_bytes()).hexdigest()
), "Do not change the executor after opening model programs"
generations = json.loads(get("hf_pilot/generation_report.json").read_text())
data = {}
for split in ["validation", "test"]:
rows = pq.read_table(
hf_hub_download(
generations["data_repo"],
f"data/{split}.parquet",
repo_type="dataset",
revision=generations["data_revision"],
)
).to_pylist()
data.update({r["id"]: r for r in rows})
ids = [r["id"] for r in generations["conditions"]["base"]]
assert len(ids) == 192 and len(set(ids)) == 192
assert ids == [r["id"] for r in generations["conditions"]["adapted"]]
versions = sandbox.initialize()
isolation = sandbox.run("", 1, mode="isolation_check")
assert isolation == preflight["isolation"]
def execute(program, row):
# Unwrap only a single full-output code fence. No prose, repairs or
# target-informed program edits are allowed.
wrapped = re.fullmatch(r"\s*```(?:python)?\s*\n(.*?)\n```\s*", program, flags=re.S)
clean = wrapped.group(1) if wrapped else program
syntax = True
imports = set()
try:
tree = ast.parse(clean)
for node in ast.walk(tree):
if isinstance(node, ast.Import):
imports.update(a.name.split(".")[0] for a in node.names)
elif isinstance(node, ast.ImportFrom):
imports.add((node.module or "").split(".")[0])
except SyntaxError:
syntax = False
result = sandbox.run(clean, row["qubits"])
fidelity = None
state_pass = False
if result.get("status") == "executed":
actual = np.array(result["real"]) + 1j * np.array(result["imag"])
target = np.array(row["state_real"]) + 1j * np.array(row["state_imag"])
fidelity = float(abs(np.vdot(target, actual)) ** 2)
state_pass = abs(result["norm"] - 1) < 1e-6 and fidelity > 1 - 1e-6
requested = row["framework"] in imports
execution = {k: v for k, v in result.items() if k not in ["real", "imag"]}
return dict(
syntax_parses=syntax,
requested_framework_import=requested,
single_fence_unwrapped=wrapped is not None,
execution=execution,
state_pass=state_pass,
fidelity=fidelity,
passed=state_pass and requested,
program_sha256=hashlib.sha256(program.encode()).hexdigest(),
)
# Every selected reference must pass the same evaluator before scoring models.
reference_checks = []
for identifier in ids:
row = data[identifier]
outcome = execute(row["program"], row)
assert outcome["passed"], (identifier, outcome)
reference_checks.append(dict(id=identifier, **outcome))
records = []
for condition, outputs in generations["conditions"].items():
for output in outputs:
row = data[output["id"]]
assert row["framework"] == output["framework"] and row["qubits"] == output["qubits"]
assert np.allclose(row["state_real"], output["target_real"], atol=1e-12, rtol=0)
assert np.allclose(row["state_imag"], output["target_imag"], atol=1e-12, rtol=0)
records.append(
dict(
condition=condition,
id=row["id"],
group_id=row["group_id"],
split=output["split"],
framework=row["framework"],
generated_tokens=output["generated_tokens"],
possible_token_cap=output["generated_tokens"] >= 512,
**execute(output["program"], row),
)
)
print(
json.dumps(
dict(
stage="scored",
condition=condition,
passed=sum(r["passed"] for r in records if r["condition"] == condition),
)
),
flush=True,
)
summary = {}
for condition in ["base", "adapted"]:
summary[condition] = {}
for split in ["validation", "test"]:
chosen = [r for r in records if r["condition"] == condition and r["split"] == split]
summary[condition][split] = dict(
n=len(chosen),
passed=sum(r["passed"] for r in chosen),
pass_at_1=float(np.mean([r["passed"] for r in chosen])),
state_correct=sum(r["state_pass"] for r in chosen),
executed=sum(r["execution"].get("status") == "executed" for r in chosen),
syntax_valid=sum(r["syntax_parses"] for r in chosen),
requested_import=sum(r["requested_framework_import"] for r in chosen),
possible_token_cap=sum(r["possible_token_cap"] for r in chosen),
)
base = {r["id"]: r for r in records if r["condition"] == "base" and r["split"] == "test"}
adapted = {r["id"]: r for r in records if r["condition"] == "adapted" and r["split"] == "test"}
keys = sorted(base)
delta = np.array([int(adapted[k]["passed"]) - int(base[k]["passed"]) for k in keys])
groups = defaultdict(list)
for i, k in enumerate(keys):
groups[base[k]["group_id"]].append(i)
indices = list(groups.values())
rng = np.random.default_rng(20261031)
boot = [
float(delta[np.concatenate([indices[i] for i in draw])].mean())
for draw in rng.integers(0, len(indices), size=(2000, len(indices)))
]
report = dict(
source_revision=revision,
executor_sha256=source_hash,
versions=versions,
isolation=isolation,
data_revision=generations["data_revision"],
reference_checks=reference_checks,
records=records,
summary=summary,
paired_test=dict(
improved=int(np.sum(delta == 1)),
regressed=int(np.sum(delta == -1)),
mean_pass_delta=float(delta.mean()),
group_bootstrap_95=np.quantile(boot, [0.025, 0.975]).tolist(),
groups=len(indices),
),
limitations="One synthetic held-out-framework experiment. Pass requires requested-framework import, supported-Python execution, unit norm and global-phase-invariant exact-state fidelity >1-1e-6. Import presence is a proxy for framework compliance, not a formal data-flow proof. Unsupported valid Python is rejected and can yield conservative false negatives. Only a full-output code fence may be removed. No repairs, retries or generated-code access to files/network. Original batch-width at_token_cap flag is unreliable and not used; generated_tokens>=512 is reported as a possible cap, not an exact stop reason.",
)
report["release_gate_pass"] = (
summary["adapted"]["test"]["passed"] > summary["base"]["test"]["passed"]
)
folder = Path("quantum_execution")
folder.mkdir(exist_ok=True)
(folder / "execution.json").write_text(json.dumps(report, indent=2))
(folder / "train_quantum_execute_pilot.py").write_bytes(Path(__file__).read_bytes())
(folder / "quantum_sandbox.py").write_bytes(Path(sandbox.__file__).read_bytes())
api.upload_folder(
repo_id=REPO,
folder_path=str(folder),
path_in_repo="execution",
commit_message="Evaluate frozen base/adapter programs under audited nobody/seccomp runner with matched reference checks and exact quantum-state fidelity",
)
print(
json.dumps(
dict(summary=summary, gate=report["release_gate_pass"], paired=report["paired_test"])
),
flush=True,
)
if __name__ == "__main__":
main()