Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Text-to-Scene: the October 2026 sandboxed re-run replaces the paper's Research Track numbers (user, 2026-10-16). | |
| The user: "这里是内部的分数，我们重新跑的，我们把分数更新到public site … 这里我们替换原来的T2S leaderboard，然后case也可以 | |
| 换成最新的，然后gallery可以放上去", then "没跑完的就不用放上去了" (leave out the runs that have not finished). | |
| Inputs (--src): a snapshot of the internal Text-to-Scene leaderboard's served folder, plus one extraction. | |
| data.json per model: its 20 public "-long" case scores (a case without a valid submission counts 0), | |
| state ("final" = all 20 scored), per-case cost and the median cost per case | |
| components.json per model and case, the Detailed / Overview / Physical scores of the result that case score | |
| comes from (scripts/t2s_components.py) | |
| gallery.json the cases' scene briefs | |
| grids/<case>-compare.{mp4,jpg} four agents building the same scene (build time-lapse, then an orbit) | |
| What it writes: | |
| - Leaderboard, Text-to-Scene tab: the bars (assets/leaderboard-bars.js), Pareto (data/pareto.json), tooltips | |
| (data/leaderboard-details.json) and the ranking table and its notes (index.html). Only "final" models. | |
| Scores follow the internal leaderboard's rule: never rounded up (bar labels x100 rounded down, table values | |
| truncated to three decimals). Cost is the median USD per case, rounded to the cent, as there; a model whose | |
| agent records no usage has no cost and no Pareto point. | |
| - Cases, Text-to-Scene tab: the comparison videos with a scene picker (assets/compare-player.js), in place of the | |
| paper round's four live scenes; the Industrial Harbor showcase stays. The case explorer button is | |
| Image-to-Scene only; Text-to-Scene links to the gallery (scripts/t2s_gallery.py). | |
| - data/t2s-sandbox.json: the public data behind all of it (no internal paths). | |
| The Image-to-Scene tab, the Task Format example and the Takeaways keep the paper's numbers. | |
| Re-runnable: every output is regenerated from --src, so a later snapshot just needs another run. | |
| python3 scripts/t2s_sandbox_round.py --src ../t2s-src [--videos blue-city,ruins,...] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import html | |
| import json | |
| import math | |
| import re | |
| import shutil | |
| import statistics | |
| import subprocess | |
| from pathlib import Path | |
| from bs4 import BeautifulSoup | |
| ROOT = Path(__file__).resolve().parent.parent | |
| V = "20261016t2s" | |
| CASES = 20 | |
| # The static Space host does not serve a folder's index.html ("gallery/" redirects to huggingface.co), so every link | |
| # names the page, as the rest of the site does with index.html#... | |
| GALLERY = "gallery/index.html" | |
| WEIGHTS = (0.2, 0.6, 0.2) # Detailed, Overview, Physical | |
| # provider -> logo key (data/site.js SITE.logos, pareto.js colours) and bar colours (the figure kit's) | |
| ORG = { | |
| "Anthropic": ("claude", "#D97757", "#ffffff"), | |
| "OpenAI": ("openai", "#10A37F", "#ffffff"), | |
| "Google": ("gemini", "#7b1fa2", "#ffffff"), | |
| "Meta": ("meta", "#42a5f5", "#111111"), | |
| "xAI": ("grok", "#111111", "#ffffff"), | |
| "Z.ai": ("zai", "#96650b", "#ffffff"), | |
| "DeepSeek": ("deepseek", "#3f51e0", "#ffffff"), | |
| "Thinking Machines": ("thinkingmachines", "#686868", "#ffffff"), | |
| "Alibaba": ("qwen", "#fe7016", "#111111"), | |
| # the internal leaderboard's amber for the bars (not in the kit); the mark is Xiaomi's own, from Simple Icons | |
| # (Lobehub has only a MiMo wordmark icon, illegible at 22 px). User, 2026-10-16: "小米 mi 标" | |
| "Xiaomi": ("xiaomi", "#f2b418", "#111111"), | |
| } | |
| # weights availability, by model family (every shown model must be listed: a new one is a decision, not a default). | |
| # Muse Spark Open and MiMo V2.6 Pro: OpenRouter's catalogue lists their Hugging Face weights | |
| # (meta-models/Muse-Spark-Open-816B-A42B, XiaomiMiMo/MiMo-V2.6-Pro-RL); user, 2026-10-16: tag both open weights. | |
| # Inkling is in neither list on purpose: OpenRouter lists its weights (thinkingmachines/Inkling) while the paper's | |
| # Image-to-Scene data tags it closed, so its Text-to-Scene run needs a decision once it is final. | |
| OPEN = ("GLM-", "DeepSeek ", "Gemma ", "Qwen ", "Muse Spark Open", "MiMo ") | |
| CLOSED = ("Claude ", "GPT-", "Gemini ", "Grok ", "Muse Spark 1", "Muse Wonder ") | |
| # scene titles (the internal pages print the case id; these are the public names) | |
| TITLES = { | |
| "abandoned-psychiatric-hospital": "Abandoned Psychiatric Hospital", "blue-city": "Blue City", | |
| "castleriver": "Castle River", "cyberpunk": "Cyberpunk", "egyptian-temple": "Egyptian Temple", | |
| "european-street-3d-scene": "European Street", "factory-environment": "Factory Environment", | |
| "hong-kong": "Hong Kong", "khs-house-of-baeryeom": "House of Baeryeom", | |
| "medieval-big-farm-town": "Medieval Big Farm Town", "medieval-buildings-volume-1": "Medieval Buildings", | |
| "nordic-harbour": "Nordic Harbour", "paris": "Paris", | |
| "post-apocalyptic-stylized-market": "Post-Apocalyptic Stylized Market", "ruins": "Ruins", | |
| "scifi-environment-offworld": "Sci-Fi Environment Offworld", "street-new-york": "New York Street", | |
| "stylized-elven-valley": "Stylized Elven Valley", "urban-decay": "Urban Decay", "winter-town": "Winter Town", | |
| } | |
| # Costs the internal table cannot read from the agent stream, recomputed from the agent's own logs (user, 2026-10-16: | |
| # "这个成本没有更新？去看看trace重新算一下"): internal key -> (file in --src from scripts/t2s_muse_cost.py, basis shown) | |
| TRACE_COSTS = { | |
| "musewf10-max-t2s20": ("musewf_cost.json", | |
| "sold by subscription; cost = the token counts its CLI logs, at Muse Wonder's per-token list price"), | |
| # self-hosted, so nothing was billed (user, 2026-10-16: "Gemma的价格可以估算一下？"); scripts/t2s_token_cost.py | |
| "gemma4-think-public95": ("gemma4think_cost.json", | |
| "self-hosted; cost = the token counts its agent logs, at Gemma 4 31B's OpenRouter list price " | |
| "($0.20 input, $0.40 output per million)"), | |
| } | |
| # the comparison videos' line-up (internal video/stack_video.py), in the order they are drawn | |
| VIDEO_MODELS = ["Claude Opus 5.5 (max)", "GPT-6 Astra (max)", "GPT-6.1 Sol (max)", "Gemini 4 Argon (high)"] | |
| VIDEO_ORDER = ["blue-city", "nordic-harbour", "cyberpunk", "medieval-big-farm-town", "factory-environment", | |
| "post-apocalyptic-stylized-market", "ruins", "scifi-environment-offworld", "egyptian-temple", | |
| "winter-town"] | |
| def one(found, what): | |
| assert found, f"not found: {what}" | |
| return found | |
| def trunc(x: float, n: int = 3) -> str: | |
| """Truncate, never round up (the internal leaderboard's rule): 0.718775 -> '0.718'.""" | |
| return f"{math.floor(x * 10 ** n + 1e-9) / 10 ** n:.{n}f}" | |
| def label(x: float) -> str: | |
| return str(math.floor(x * 100 + 1e-9)) | |
| def slug(name: str) -> str: | |
| return re.sub(r"[^a-z0-9.]+", "-", name.lower()).strip("-") | |
| def case_slug(case_id: str) -> str: | |
| return case_id[:-5] if case_id.endswith("-long") else case_id | |
| def weights_open(name: str) -> bool: | |
| if name.startswith(OPEN): | |
| return True | |
| assert name.startswith(CLOSED), f"{name}: add it to OPEN or CLOSED" | |
| return False | |
| # ---- the data ------------------------------------------------------------------------------------------------ | |
| def load(src: Path) -> dict: | |
| data = json.loads((src / "data.json").read_text()) | |
| comps = {m["key"]: {c["case"]: c for c in m["cases"]} for m in json.loads((src / "components.json").read_text())["models"]} | |
| briefs = {c["id"]: c["brief"] for c in json.loads((src / "gallery.json").read_text())["cases"]} | |
| cases = data["cases"] | |
| assert len(cases) == CASES and all(c in briefs for c in cases), "case list" | |
| models, left_out = [], [] | |
| for m in data["models"]: | |
| per = {c["case"]: c for c in m["cases"]} | |
| final = m["state"] == "final" and m["scored"] == m["of"] == CASES and all( | |
| per.get(c, {}).get("status") == "scored" and isinstance(per[c].get("score"), (int, float)) for c in cases) | |
| if not final: | |
| left_out.append(f"{m['name']}: {m['scored']}/{m['of']} scored ({m['state']})") | |
| continue | |
| assert m["org"] in ORG, f"{m['org']}: add the provider to ORG" | |
| rows, cc = {}, comps.get(m["key"]) or {} | |
| traced, basis = None, None | |
| if m.get("cost_median_usd") is None and m["key"] in TRACE_COSTS and (src / TRACE_COSTS[m["key"]][0]).exists(): | |
| traced = json.loads((src / TRACE_COSTS[m["key"]][0]).read_text())["cases"] | |
| assert all(traced.get(c, {}).get("cost_usd") is not None for c in cases), f"{m['name']}: traced costs" | |
| basis = TRACE_COSTS[m["key"]][1] | |
| for c in cases: | |
| s = float(per[c]["score"]) | |
| got = cc.get(c) or {} | |
| if s == 0: # no valid scene, or a failed content check: every component 0 | |
| d = o = p = 0.0 | |
| else: | |
| comp = one(got.get("components"), f"{m['name']} / {c}: components") | |
| assert abs(float(got.get("score")) - s) < 1e-9, f"{m['name']} / {c}: score {got.get('score')} != {s}" | |
| d, o, p = (float(comp[k] or 0) for k in ("detailed", "overview", "physical")) | |
| w = sum(a * b for a, b in zip(WEIGHTS, (d, o, p))) | |
| assert abs(w - s) < 2.5e-4, f"{m['name']} / {c}: 0.2D+0.6O+0.2P = {w:.5f}, score {s}" | |
| cost = ((m.get("cost_cases") or {}).get(c) or {}).get("cost_usd") | |
| if cost is None and traced: | |
| cost = traced[c]["cost_usd"] | |
| rows[c] = {"score": s, "detailed": d, "overview": o, "physical": p, | |
| "cost": round(cost, 6) if isinstance(cost, (int, float)) else None} | |
| mean = sum(r["score"] for r in rows.values()) / CASES | |
| assert abs(mean - m["mean"]) < 1e-9, f"{m['name']}: mean {mean} != {m['mean']}" | |
| med = m.get("cost_median_usd") | |
| costs = [r["cost"] for r in rows.values() if r["cost"] is not None] | |
| if med is None and traced: | |
| med = statistics.median(costs) | |
| if med is not None: | |
| assert len(costs) == CASES and abs(statistics.median(costs) - med) < 1e-3, f"{m['name']}: median cost" | |
| # why a model has no cost, as the pages say it: a self-hosted run reports $0 (the internal table drops it) | |
| nocost = None if med is not None else ( | |
| "it ran on self-hosted weights, so nothing was billed" | |
| if any("unbilled" in s for s in m.get("cost_source") or []) else "its agent records no token usage") | |
| logo, colour, ink = ORG[m["org"]] | |
| models.append({ | |
| "key": slug(m["name"]), "name": m["name"], "org": m["org"], "logo": logo, "colour": colour, "ink": ink, | |
| "open": weights_open(m["name"]), "score": round(mean, 6), | |
| "detailed": round(sum(r["detailed"] for r in rows.values()) / CASES, 6), | |
| "overview": round(sum(r["overview"] for r in rows.values()) / CASES, 6), | |
| "physical": round(sum(r["physical"] for r in rows.values()) / CASES, 6), | |
| "cost": round(med, 6) if med is not None else None, "cost_basis": basis, "cost_none": nocost, "cases": rows, | |
| }) | |
| models.sort(key=lambda r: -r["score"]) | |
| priced = [r for r in models if r["cost"] is not None] | |
| for a in models: | |
| a["front"] = a["cost"] is not None and not any( | |
| b["cost"] <= a["cost"] and b["score"] >= a["score"] and (b["cost"] < a["cost"] or b["score"] > a["score"]) | |
| for b in priced) | |
| return {"when": data["when"], "cases": cases, "briefs": briefs, "models": models, "left_out": left_out} | |
| def write_data(D: dict) -> None: | |
| out = { | |
| "track": "Text-to-Scene · Research Track (20 public scenes) · sandboxed re-run", | |
| "when": D["when"], | |
| "score": "case score = 0.2 x Detailed + 0.6 x Overview + 0.2 x Physical; a case without a valid scene scores 0; " | |
| "a model's score is the mean over the 20 cases", | |
| "cost": "median USD per case over the model's 20 cases; null when the agent records no usage or the run was " | |
| "self-hosted (nothing billed), cost_none says which", | |
| "cases": [{"id": c, "slug": case_slug(c), "title": TITLES[case_slug(c)], "brief": D["briefs"][c]} for c in D["cases"]], | |
| "models": [{k: m[k] for k in ("key", "name", "org", "logo", "colour", "open", "score", "detailed", "overview", "physical", "cost", | |
| "cost_basis", "cost_none", | |
| "front", "cases")} for m in D["models"]], | |
| } | |
| (ROOT / "data/t2s-sandbox.json").write_text(json.dumps(out, indent=1, ensure_ascii=False) + "\n") | |
| def write_bars(D: dict) -> None: | |
| p = ROOT / "assets/leaderboard-bars.js" | |
| s = p.read_text() | |
| m = one(re.search(r"const ALL = (\{.*?\});\n", s, re.S), "ALL") | |
| ALL = json.loads(m.group(1)) | |
| marks = json.loads(one(re.search(r"const LOGOS = (\{.*?\});\n", s, re.S), "LOGOS").group(1)) | |
| for r in D["models"]: # the bar draws the provider's mark under it, from LOGOS | |
| assert r["org"] in marks, f"{r['org']}: add its mark to LOGOS in assets/leaderboard-bars.js" | |
| ALL["t2s"] = [{"name": r["name"], "org": r["org"], "score": r["score"], "colour": r["colour"], "ink": r["ink"], | |
| "label": label(r["score"]), "new": False} for r in D["models"]] | |
| s = s[:m.start(1)] + json.dumps(ALL, ensure_ascii=False) + s[m.end(1):] | |
| s = re.sub(r"chart-interactions\.js\?v=\w+", f"chart-interactions.js?v={V}", s) | |
| p.write_text(s) | |
| def write_pareto(D: dict) -> None: | |
| p = ROOT / "data/pareto.json" | |
| d = json.loads(p.read_text()) | |
| colours = one(re.search(r"const colors=\{(.*?)\};", (ROOT / "assets/pareto.js").read_text()), "pareto colours").group(1) | |
| for r in D["models"]: # a key missing there is drawn in the fallback grey | |
| assert r["cost"] is None or re.search(rf"\b{r['logo']}:'#", colours), f"{r['logo']}: add its colour to assets/pareto.js" | |
| d["t2s"] = [{"key": r["key"], "name": r["name"], "logo": r["logo"], "open": r["open"], "score": r["score"], | |
| "cost": r["cost"]} for r in D["models"] if r["cost"] is not None] | |
| p.write_text(json.dumps(d, indent=2, ensure_ascii=False) + "\n") | |
| def write_details(D: dict) -> None: | |
| p = ROOT / "data/leaderboard-details.json" | |
| d = json.loads(p.read_text()) | |
| d["t2s"] = [{"key": r["key"], "name": r["name"], "provider": r["org"], "metrics": [ | |
| {"label": "Text-to-Scene", "value": r["score"], "currency": False}, | |
| {"label": "Detailed Alignment", "value": r["detailed"], "currency": False}, | |
| {"label": "Overview Alignment", "value": r["overview"], "currency": False}, | |
| {"label": "Physical Safety", "value": r["physical"], "currency": False}, | |
| {"label": "Median cost / case", "value": f"{r['cost']:.2f}" if r["cost"] is not None else | |
| "not billed (self-hosted)" if "self-hosted" in r["cost_none"] else "not recorded", | |
| "currency": r["cost"] is not None}]} for r in D["models"]] | |
| p.write_text(json.dumps(d, indent=2, ensure_ascii=False) + "\n") | |
| # ---- index.html ------------------------------------------------------------------------------------------------ | |
| def site_logos() -> dict: | |
| out = subprocess.run(["node", "-e", "global.window={};eval(require('fs').readFileSync(process.argv[1],'utf8'));" | |
| "process.stdout.write(JSON.stringify(window.SITE.logos))", str(ROOT / "data/site.js")], | |
| capture_output=True, text=True, check=True).stdout | |
| return json.loads(out) | |
| def table_rows(D: dict) -> str: | |
| logos, top, rows = site_logos(), D["models"][0]["score"], [] | |
| for i, r in enumerate(D["models"]): | |
| svg = logos[r["logo"]] | |
| for gid in set(re.findall(r'id="([^"]+)"', svg)): # one gradient per row | |
| svg = svg.replace(f'id="{gid}"', f'id="{gid}T{i}"').replace(f"url(#{gid})", f"url(#{gid}T{i})") | |
| tags = ('<span class="tag open">open weights</span>' if r["open"] else "") + \ | |
| ('<span class="tag front">Pareto</span>' if r["front"] else "") | |
| tip = f' title="{html.escape(r["name"])}: {html.escape(r["cost_basis"])}"' if r["cost_basis"] else "" | |
| cost = (f'<td class="dim" data-v="{r["cost"]}"{tip}>${r["cost"]:.2f}{"*" if r["cost_basis"] else ""}</td>' if r["cost"] is not None else | |
| f'<td class="dim" data-v="" title="{html.escape(r["name"])}: {html.escape(r["cost_none"])}">–</td>') | |
| rows.append( | |
| f'<tr data-model-key="{r["key"]}" data-search="{html.escape((r["name"] + " " + r["org"]).lower())}">' | |
| f'<td class="rk">{i + 1}</td><td class="nm"><span class="logo">{svg}</span>{html.escape(r["name"])}{tags}' | |
| f'<span class="org">{html.escape(r["org"])}</span></td>' | |
| f'<td data-v="{r["score"]}"><div class="cellbar"><i style="width:{r["score"] / top * 80:.1f}px;--bar:var(--c4s-a)"></i>' | |
| f'<b>{trunc(r["score"])}</b></div></td>' | |
| f'<td class="dim" data-v="{r["detailed"]}">{trunc(r["detailed"])}</td>' | |
| f'<td class="dim" data-v="{r["overview"]}">{trunc(r["overview"])}</td>' | |
| f'<td class="dim" data-v="{r["physical"]}">{trunc(r["physical"])}</td>{cost}</tr>') | |
| return "".join(rows) | |
| def by_view(tag, view: str): | |
| return tag.get("data-view") == view | |
| def leaderboard(soup, D: dict) -> None: | |
| frag = lambda s: BeautifulSoup(s, "html.parser") | |
| n = len(D["models"]) | |
| lb = one(soup.find(id="leaderboard"), "leaderboard") | |
| one(lb.find("p", class_="section-desc", attrs={"data-view": "t2s"}), "t2s desc").string = \ | |
| f"Research Track · 20 scenes · {n} coding-agent configurations · re-run in October 2026." | |
| one(lb.find("p", class_="bar-mobile-hint"), "mobile hint").string = "Swipe to view every configuration →" | |
| one(lb.find("div", class_="bar-scroll"), "bar scroll")["aria-label"] = "All model configurations" | |
| one(lb.find(id="bars-t2s"), "bars-t2s")["aria-label"] = f"Text-to-Scene scores for the {n} Code4Scene configurations" | |
| # the charts name the task, not the track (user, 2026-10-17: "Research Track？不要这个吧 … 下面那个图上") | |
| one(lb.select_one('.bar-heading h3 span.q[data-view="t2s"]'), "bar qualifier").string = "Text-to-Scene" | |
| # the bar note: one per view | |
| card = one(lb.find(class_="bar-card"), "bar card") | |
| for note in card.find_all("p", class_="note"): | |
| note.decompose() | |
| card.append(frag( | |
| '<p class="note" data-view="t2s">Sorted by score. Bar labels are scores ×100, rounded down; the table truncates ' | |
| 'to three decimal places. One color per model provider.</p>' | |
| '<p class="note" data-view="i2s" style="display:none">Sorted by score. Bar labels round scores ×100; the table ' | |
| 'preserves three decimal places. One color per model provider.</p>')) | |
| # the Pareto: median cost for this tab | |
| pc = one(lb.find(class_="pareto-card"), "pareto card") | |
| one(pc.find("p", class_="figure-subtitle", attrs={"data-view": "t2s"}), "pareto t2s subtitle").string = \ | |
| "Quality against the median cost of one case · Text-to-Scene, 20 scenes" | |
| one(pc.find("svg", attrs={"data-mode": "t2s"}), "pareto t2s svg")["aria-label"] = \ | |
| "t2s score against median cost per case. More expensive runs on the left; logarithmic scale." | |
| for cap in pc.find_all("p", class_="figure-caption"): | |
| cap.decompose() | |
| miss = "".join(f" {r['name']} has no point: {r['cost_none']}." for r in D["models"] if r["cost"] is None) | |
| basis ="".join(f" {r['name']} is {r['cost_basis']}." for r in D["models"] if r["cost_basis"]) | |
| pc.append(frag( | |
| '<p class="figure-caption" data-view="t2s">Solid: the frontier — nothing cheaper scores higher. One point per ' | |
| f'configuration, at its evaluated reasoning effort. Cost is the median USD per case.{html.escape(miss + basis)}</p>' | |
| '<p class="figure-caption" data-view="i2s" style="display:none">Solid: the frontier — nothing cheaper scores ' | |
| 'higher. One point per configuration, at its evaluated reasoning effort. Cost is the mean USD per case.</p>')) | |
| # the ranking | |
| one(lb.find(id="ranking-count"), "ranking count").string = f"{n} configurations" | |
| t2s = one([d for d in lb.find_all("div", attrs={"data-view": "t2s"}) if d.find("table", class_="board")], "t2s ranking")[0] | |
| one(t2s.find("p", class_="board-note"), "t2s board note").string = \ | |
| ("Research Track, 20 scenes, re-run in October 2026 with every agent in the same sandbox. Case score = 0.2 × " | |
| "Detailed + 0.6 × Overview + 0.2 × Physical; a case without a valid scene scores 0. Detailed, Overview and " | |
| "Physical are means of the per-case scores. Cost is the median per case." | |
| + "".join(f" *{r['name']} is {r['cost_basis']}." for r in D["models"] if r["cost_basis"])) | |
| th = one(t2s.find("th", attrs={"data-k": "c_t2s"}), "t2s cost header") | |
| th.string = "Median cost" | |
| tbody = one(t2s.find("table", class_="board").tbody, "t2s tbody") | |
| tbody.clear() | |
| tbody.append(frag(table_rows(D))) | |
| rc = one(lb.find(class_="ranking-card"), "ranking card") | |
| for old in rc.find_all("p", class_="gallery-link"): | |
| old.decompose() | |
| rc.append(frag(f'<p class="board-note gallery-link" data-view="t2s"><a href="{GALLERY}">Open the gallery</a>: what ' | |
| f'each of the {n} configurations built on every scene, with its score and cost.</p>')) | |
| def compare_card(D: dict, videos: list[str]) -> str: | |
| byname = {m["name"]: m for m in D["models"]} | |
| for name in VIDEO_MODELS: | |
| one(byname.get(name), f"video model {name} on the leaderboard") | |
| items = [] | |
| for cs in videos: | |
| cid = cs + "-long" | |
| brief = D["briefs"][cid] | |
| items.append({"id": cs, "title": TITLES[cs], "brief": brief, | |
| "mp4": f"media/t2s/compare/{cs}.mp4?v={V}", "poster": f"media/t2s/compare/{cs}.jpg?v={V}", | |
| "gallery": f"gallery/cases.html#{cs}", | |
| "scores": [[n, trunc(byname[n]["cases"][cid]["score"])] for n in VIDEO_MODELS]}) | |
| first = items[0] | |
| picker = "".join(f'<button type="button" role="tab" aria-selected="{str(i == 0).lower()}" data-i="{i}"' | |
| + ("" if i == 0 else ' tabindex="-1"') | |
| + f' aria-controls="cmp-stage">{html.escape(it["title"])}</button>' | |
| for i, it in enumerate(items)) | |
| scores = "".join(f'<li><span>{html.escape(n)}</span><b>{s}</b></li>' for n, s in first["scores"]) | |
| payload = json.dumps(items, ensure_ascii=False).replace("</", "<\\/") | |
| return ( | |
| '<article class="case-card case-compare" id="t2s-compare">' | |
| f'<div aria-label="Scene" class="cmp-picker" role="tablist">{picker}</div>' | |
| '<div class="cmp-stage" id="cmp-stage" role="tabpanel"><div class="cmp-scroll" role="region" tabindex="0" ' | |
| 'aria-label="Comparison video, scrollable on small screens">' | |
| f'<video class="cmp-video" controls muted loop playsinline preload="metadata" poster="{first["poster"]}" ' | |
| f'src="{first["mp4"]}" aria-label="{html.escape(first["title"])}: four agents build the scene, then an orbit ' | |
| 'of each finished scene"></video></div><p class="cmp-swipe">Swipe to see all four agents →</p></div>' | |
| '<div class="meta cmp-meta"><span class="eyebrow">Text-to-Scene · re-run, October 2026</span>' | |
| f'<h4 id="cmp-title">{html.escape(first["title"])}</h4>' | |
| f'<p class="cmp-brief" id="cmp-brief">{html.escape(first["brief"])}</p>' | |
| f'<ul class="cmp-scores" id="cmp-scores" aria-label="Case scores">{scores}</ul>' | |
| f'<a class="text-button" href="{first["gallery"]}" id="cmp-gallery">Compare all {len(D["models"])} agents on this scene ↗</a>' | |
| '<p class="feature-note">Each tile: the build as a time-lapse, then an orbit of the scene the agent saved. ' | |
| 'Chips: the case score ×100, rounded down.</p></div>' | |
| f'<script id="cmp-data" type="application/json">{payload}</script>' | |
| '</article>') | |
| def cases_section(soup, D: dict, videos: list[str]) -> None: | |
| frag = lambda s: BeautifulSoup(s, "html.parser") | |
| sec = one(soup.find(id="cases"), "cases") | |
| one(sec.find("p", class_="section-desc", attrs={"data-view": "t2s"}), "cases t2s desc").string = \ | |
| (f"Four agents build the same scene: a time-lapse of each build, then an orbit of the result. Pick a scene, or open " | |
| f"the gallery for all {len(D['models'])} configurations on all 20 scenes.") | |
| grid = one(sec.find("div", class_="case-grid", attrs={"data-view": "t2s"}), "t2s case grid") | |
| feature = one(grid.find("article", class_="case-feature"), "Industrial Harbor showcase") | |
| feature.extract() | |
| grid.clear() | |
| grid.append(frag(compare_card(D, videos))) | |
| grid.append(feature) | |
| # the explorer button: Image-to-Scene's; Text-to-Scene gets the gallery | |
| btn = one(sec.find("a", class_="btn", href="cases.html"), "case explorer button") | |
| wrap = btn.parent | |
| wrap["data-view"] = "i2s" | |
| wrap["style"] = "text-align:center;margin-top:2rem;display:none" | |
| for old in sec.find_all("div", class_="gallery-cta"): | |
| old.decompose() | |
| gal = frag('<div class="gallery-cta" data-view="t2s" style="text-align:center;margin-top:2rem"><a class="btn btn-primary" ' | |
| f'href="{GALLERY}"><svg aria-hidden="true" fill="none" stroke="currentColor" stroke-width="1.5" viewbox="0 0 16 16">' | |
| '<rect height="5" rx="1" width="5" x="2" y="2"></rect><rect height="5" rx="1" width="5" x="9" y="2"></rect>' | |
| '<rect height="5" rx="1" width="5" x="2" y="9"></rect><rect height="5" rx="1" width="5" x="9" y="9"></rect>' | |
| '</svg> Open the gallery</a></div>') | |
| wrap.insert_before(gal) | |
| def takeaways(soup) -> None: | |
| tk = one(soup.find(id="takeaways"), "takeaways") | |
| one(tk.find("p", class_="section-desc"), "takeaways desc").string = \ | |
| ("What the paper's evaluation shows across its 14 agents. Every chart below is drawn from the paper's numbers, " | |
| "not the October 2026 Text-to-Scene re-run above; hover for values.") | |
| def nav(soup, prefix: str = "") -> None: | |
| n = one(soup.find("nav", attrs={"aria-label": "Main navigation"}), "main nav") | |
| old = n.find("a", href=(prefix + "gallery/", prefix + GALLERY)) | |
| if old: | |
| old["href"] = prefix + GALLERY | |
| return | |
| vis = one(n.find("a", href=re.compile(r"index\.html#cases$")), "Visualization link") | |
| a = soup.new_tag("a", href=prefix + GALLERY) | |
| a.string = "Gallery" | |
| vis.insert_after(a) | |
| def scripts_and_styles(soup) -> None: | |
| for t in soup.find_all(["script", "link"]): | |
| a = "src" if t.name == "script" else "href" | |
| u = t.get(a) or "" | |
| if re.match(r"assets/(buildingbench\.css|leaderboard-bars\.js|pareto\.js|leaderboard-controls\.js|" | |
| r"takeaway-bars\.js|paper\.js)", u): | |
| t[a] = re.sub(r"\?v=.*$", "", u) + f"?v={V}" | |
| if not soup.find("script", src=re.compile(r"assets/compare-player\.js")): | |
| soup.body.append(soup.new_tag("script", attrs={"type": "module", "src": f"assets/compare-player.js?v={V}"})) | |
| def index(D: dict, videos: list[str]) -> None: | |
| p = ROOT / "index.html" | |
| soup = BeautifulSoup(p.read_text(), "html.parser") | |
| leaderboard(soup, D) | |
| cases_section(soup, D, videos) | |
| takeaways(soup) | |
| nav(soup) | |
| scripts_and_styles(soup) | |
| p.write_text(str(soup)) | |
| def other_pages() -> None: | |
| """The gallery in the main navigation; the case explorer says which round its scenes are from.""" | |
| for name in ("cases.html",): | |
| p = ROOT / name | |
| soup = BeautifulSoup(p.read_text(), "html.parser") | |
| nav(soup) | |
| if name == "cases.html": | |
| desc = one(soup.find("p", class_="section-desc"), "explorer desc") | |
| old = desc.find("a", href=("gallery/", GALLERY)) | |
| if old: | |
| old["href"] = GALLERY | |
| else: | |
| desc.append(BeautifulSoup( | |
| ' These are the paper’s scenes and scores. For the October 2026 Text-to-Scene re-run, every agent’s ' | |
| f'scene on every case is in the <a href="{GALLERY}">gallery</a>.', "html.parser")) | |
| for t in soup.find_all("script", src=re.compile(r"^assets/paper\.js")): | |
| t["src"] = f"assets/paper.js?v={V}" | |
| p.write_text(str(soup)) | |
| def media(src: Path, videos: list[str]) -> None: | |
| out = ROOT / "media/t2s/compare" | |
| out.mkdir(parents=True, exist_ok=True) | |
| for f in out.iterdir(): | |
| if f.stem not in videos: | |
| f.unlink() | |
| for cs in videos: | |
| for ext in ("mp4", "jpg"): | |
| s = one((src / "grids" / f"{cs}-compare.{ext}").exists() and src / "grids" / f"{cs}-compare.{ext}", f"{cs} {ext}") | |
| if not (out / f"{cs}.{ext}").exists() or (out / f"{cs}.{ext}").read_bytes() != s.read_bytes(): | |
| shutil.copyfile(s, out / f"{cs}.{ext}") | |
| CSS_START, CSS_END = "/* t2s re-run: comparison videos, gallery link (scripts/t2s_sandbox_round.py) */", "/* end t2s re-run */" | |
| CSS = """ | |
| .case-grid .case-compare{grid-column:1/-1;display:flex;flex-direction:column} | |
| .cmp-picker{display:flex;flex-wrap:wrap;gap:6px;padding:14px 16px;border-bottom:1px solid var(--line-soft)} | |
| .cmp-picker button{font:500 12px/1.2 Inter,Arial,sans-serif;padding:7px 12px;border:1px solid var(--line);border-radius:999px;background:#fff;color:var(--text-p);cursor:pointer;transition:border-color .15s,background .15s,color .15s} | |
| .cmp-picker button:hover{border-color:var(--green);color:var(--green)} | |
| .cmp-picker button[aria-selected=true]{background:var(--green-dark);border-color:var(--green-dark);color:#fff} | |
| .cmp-picker button:focus-visible{outline:2px solid var(--green);outline-offset:2px} | |
| .cmp-scroll{overflow-x:auto;background:#fff} | |
| .cmp-scroll:focus-visible{outline:2px solid var(--green);outline-offset:-2px} | |
| .cmp-video{display:block;width:100%;height:auto;aspect-ratio:1920/588;background:#fff} | |
| .cmp-swipe{display:none;margin:8px 16px 0;font-size:11px;color:var(--text-muted)} | |
| .case-card .meta.cmp-meta{border-top:1px solid var(--line-soft)} | |
| .case-card .meta.cmp-meta h4{font-size:17px} | |
| .case-card .meta p.cmp-brief{max-width:980px;margin-top:2px;font-size:12.5px;line-height:1.6;color:var(--text-p);display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden} | |
| .case-card .meta p.cmp-brief.open{display:block} | |
| .cmp-more{margin-top:4px;padding:0;border:0;background:none;font:600 11px Inter,Arial,sans-serif;color:var(--green);cursor:pointer} | |
| .cmp-scores{list-style:none;display:flex;flex-wrap:wrap;gap:6px 22px;margin:12px 0 0;padding:0;font-size:12px;color:var(--text-p)} | |
| .cmp-scores b{margin-left:7px;font:500 12.5px 'IBM Plex Mono',monospace;color:var(--text-h)} | |
| .gallery-link a{color:var(--green);font-weight:600} | |
| @media(max-width:760px){.cmp-video{width:760px;max-width:none}.cmp-swipe{display:block}.case-card .meta p.cmp-brief{font-size:11.5px} | |
| .cmp-picker{flex-wrap:nowrap;overflow-x:auto;scrollbar-width:none;padding:12px}.cmp-picker::-webkit-scrollbar{display:none}.cmp-picker button{flex:none}} | |
| @media(max-width:650px){.site-header nav{gap:10px}} /* six items since the gallery joined the menu */ | |
| """ | |
| def css() -> None: | |
| p = ROOT / "assets/buildingbench.css" | |
| s = p.read_text() | |
| block = CSS_START + CSS + CSS_END | |
| if CSS_START in s: | |
| s = s[:s.index(CSS_START)] + block + s[s.index(CSS_END) + len(CSS_END):] | |
| else: | |
| s = s.rstrip("\n") + "\n" + block + "\n" | |
| p.write_text(s) | |
| def other_assets() -> None: | |
| """Version keys for the files this change edits, so cached copies are not mixed with new ones.""" | |
| for name, pat in (("assets/pareto.js", r"(data/pareto\.json\?v=)\w+"), | |
| ("assets/chart-interactions.js", r"(leaderboard-details\.json\?v=)\w+")): | |
| p = ROOT / name | |
| s, k = re.subn(pat, rf"\g<1>{V}", p.read_text()) | |
| assert k == 1, name | |
| p.write_text(s) | |
| for name in ("assets/pareto.js", "assets/takeaway-bars.js"): | |
| p = ROOT / name | |
| s, k = re.subn(r"chart-interactions\.js\?v=\w+", f"chart-interactions.js?v={V}", p.read_text()) | |
| assert k == 1, name | |
| p.write_text(s) | |
| # sortable tables: a cell without a value (a cost that was not recorded) sorts last in either direction | |
| p = ROOT / "assets/paper.js" | |
| s = p.read_text() | |
| old = ('rows.sort(function (a, b) { var x = +a.cells[ci].getAttribute("data-v"), y = +b.cells[ci].getAttribute("data-v"); ' | |
| 'return asc ? x - y : y - x; });') | |
| new = ('rows.sort(function (a, b) { var p = a.cells[ci].getAttribute("data-v"), q = b.cells[ci].getAttribute("data-v"); ' | |
| 'if (p === "" || q === "") return (p === "") - (q === ""); var x = +p, y = +q; return asc ? x - y : y - x; });') | |
| assert old in s or new in s, "paper.js sort" | |
| p.write_text(s.replace(old, new)) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) | |
| ap.add_argument("--src", type=Path, required=True) | |
| ap.add_argument("--videos", default=",".join(VIDEO_ORDER), help="comparison videos to show, in order") | |
| a = ap.parse_args() | |
| videos = [v for v in a.videos.split(",") if v] | |
| assert videos and all(v in VIDEO_ORDER for v in videos), videos | |
| D = load(a.src) | |
| write_data(D) | |
| write_bars(D) | |
| write_pareto(D) | |
| write_details(D) | |
| media(a.src, videos) | |
| index(D, videos) | |
| other_pages() | |
| css() | |
| other_assets() | |
| print(f"snapshot {D['when']}: {len(D['models'])} final configurations") | |
| for r in D["models"]: | |
| print(f" {r['name']:<28} {trunc(r['score'])} D {trunc(r['detailed'])} O {trunc(r['overview'])} " | |
| f"P {trunc(r['physical'])} {'$%.2f' % r['cost'] if r['cost'] is not None else '–':>7}" | |
| f"{' Pareto' if r['front'] else ''}{' open' if r['open'] else ''}") | |
| for x in D["left_out"]: | |
| print(" left out:", x) | |
| print("videos:", ", ".join(videos)) | |
| if __name__ == "__main__": | |
| main() | |