""" real_vision.py --------------- Real-photo detection for the "Visual Inventory Scan" tab. Two detector modes, both from the Ultralytics YOLO family: 1. **Standard (COCO-pretrained YOLOv8)** -- fast, but COCO's 80 classes do NOT include pallet, forklift, cardboard box or rack. It reliably finds people and vehicles (useful for safety checks) but cannot count pallets. The UI says so explicitly rather than letting a user assume otherwise. 2. **Open-vocabulary (YOLOE)** -- you type what to look for ("pallet, cardboard box, forklift") and it searches for those names zero-shot, no warehouse-specific training. Accuracy on warehouse imagery is NOT measured here (no labelled real warehouse photos were available), so treat counts as a starting point, not ground truth. Why YOLOE rather than YOLO-World: both are open-vocabulary YOLO models, but YOLO-World's text encoder (CLIP) is hosted on a CDN that could not be reached from the development environment, so it could not be verified end to end. YOLOE's text encoder (MobileCLIP) ships from the same GitHub releases as the weights, and it was tested on a real photo before shipping. Design notes ------------ * Weights are downloaded on first use from the official Ultralytics GitHub release assets and cached; nothing is bundled in the repo. * `ultralytics`/`torch` are imported lazily, so if they're unavailable the rest of the app still works and this tab shows a clear message. * Example photos are fetched at runtime from Wikimedia Commons (openly licensed) and shown with author + license. They are never bundled, so the repo contains no third-party images. If the download fails, the user is told to upload their own photo. * Uploaded images are processed in memory and not stored. """ import hashlib import os import tempfile import threading import time from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple import requests from PIL import Image, ImageDraw, ImageFont WEIGHTS_DIR = os.environ.get("YOLO_WEIGHTS_DIR", os.path.join(tempfile.gettempdir(), "yolo_weights")) EXAMPLES_DIR = os.path.join(tempfile.gettempdir(), "warehouse_examples") ASSETS_BASE = "https://github.com/ultralytics/assets/releases/download/v8.3.0/" USER_AGENT = "SmartWarehouseAIDemo/1.0 (portfolio demo; Hugging Face Space)" MAX_SIDE = 1600 # downscale very large uploads before inference (CPU friendliness) MODEL_CHOICES = { "YOLOv8n (fastest)": "yolov8n.pt", "YOLOv8s (more accurate)": "yolov8s.pt", } OPEN_VOCAB_WEIGHTS = "yoloe-11s-seg.pt" TEXT_ENCODER_WEIGHTS = "mobileclip_blt.ts" # ~600 MB, downloaded once on first open-vocabulary use DEFAULT_PROMPTS = "pallet, cardboard box, forklift, person, shelving rack" # COCO classes that matter in a warehouse -> a plain-language group WAREHOUSE_GROUPS = { "person": "people", "truck": "vehicles", "car": "vehicles", "bus": "vehicles", "motorcycle": "vehicles", "bicycle": "vehicles", "train": "vehicles", } # Openly-licensed real photos from Wikimedia Commons (fetched at runtime, never bundled). EXAMPLE_PHOTOS: Dict[str, dict] = { "Pallet-rack warehouse with forklift": { "file": "Modern_warehouse_with_pallet_rack_storage_system.jpg", "author": "Axisadman", "license": "CC BY-SA 3.0", "license_url": "https://creativecommons.org/licenses/by-sa/3.0/", }, "Warehouse floor: pallets and forklift driver": { "file": "Allentown_Project_042.jpg", "author": "Chrisking", "license": "CC BY-SA 3.0", "license_url": "https://creativecommons.org/licenses/by-sa/3.0/", }, "Stacked CHEP pallets": { "file": "576_CHEP_pallets.jpg", "author": "train_photos (via Flickr)", "license": "CC BY-SA 2.0", "license_url": "https://creativecommons.org/licenses/by-sa/2.0/", }, } @dataclass class Detection: label: str conf: float box: Tuple[int, int, int, int] # x1, y1, x2, y2 @dataclass class RealDetectionResult: annotated: Image.Image detections: List[Detection] counts: Dict[str, int] inference_ms: float model_name: str notes: List[str] = field(default_factory=list) # -------------------------------------------------------------------------- # Weights + model cache # -------------------------------------------------------------------------- _MODEL_LOCK = threading.Lock() _MODELS: Dict[str, object] = {} def _ensure_weights(filename: str) -> str: """Download an Ultralytics release asset once and cache it.""" os.makedirs(WEIGHTS_DIR, exist_ok=True) path = os.path.join(WEIGHTS_DIR, filename) if os.path.exists(path) and os.path.getsize(path) > 1_000_000: return path tmp = path + ".part" with requests.get(ASSETS_BASE + filename, stream=True, timeout=60, headers={"User-Agent": USER_AGENT}) as r: r.raise_for_status() with open(tmp, "wb") as f: for chunk in r.iter_content(chunk_size=1 << 20): f.write(chunk) os.replace(tmp, path) return path def vision_stack_available() -> Tuple[bool, str]: """Is the optional detector stack (ultralytics + torch) importable?""" try: import ultralytics # noqa: F401 return True, "" except Exception as e: # noqa: BLE001 return False, f"{type(e).__name__}: {e}" def _quiet_settings(): """Uploaded photos are user data: turn off Ultralytics' anonymous usage sync.""" try: from ultralytics import settings settings.update({"sync": False, "weights_dir": WEIGHTS_DIR}) except Exception: # noqa: BLE001 pass def _get_yolo(weights_name: str): with _MODEL_LOCK: if weights_name not in _MODELS: from ultralytics import YOLO _quiet_settings() _MODELS[weights_name] = YOLO(_ensure_weights(weights_name)) return _MODELS[weights_name] def _get_open(prompts: Tuple[str, ...]): """YOLOE with a given vocabulary. set_classes() re-encodes the text prompts, so cache one model per distinct prompt set (a few at most).""" key = "open::" + "|".join(prompts) with _MODEL_LOCK: if key not in _MODELS: from ultralytics import YOLOE _quiet_settings() # make sure the ~600 MB text encoder is fetched from the official release # assets into our cache dir, where ultralytics will look for it _ensure_weights(TEXT_ENCODER_WEIGHTS) model = YOLOE(_ensure_weights(OPEN_VOCAB_WEIGHTS)) model.set_classes(list(prompts), model.get_text_pe(list(prompts))) _MODELS[key] = model # keep the cache from growing without bound if people try many prompt sets open_keys = [k for k in _MODELS if k.startswith("open::")] for old in open_keys[:-3]: _MODELS.pop(old, None) return _MODELS[key] # -------------------------------------------------------------------------- # Drawing # -------------------------------------------------------------------------- _PALETTE = [(231, 76, 60), (52, 152, 219), (46, 204, 113), (241, 196, 15), (155, 89, 182), (230, 126, 34), (26, 188, 156), (233, 30, 99)] def _color_for(label: str) -> Tuple[int, int, int]: h = int(hashlib.md5(label.encode()).hexdigest(), 16) return _PALETTE[h % len(_PALETTE)] def _font(size: int): try: return ImageFont.load_default(size=size) # Pillow >= 10.1 except TypeError: return ImageFont.load_default() def _annotate(image: Image.Image, dets: List[Detection]) -> Image.Image: out = image.convert("RGB").copy() draw = ImageDraw.Draw(out) w, h = out.size thick = max(2, round(max(w, h) / 400)) font = _font(max(12, round(max(w, h) / 55))) for d in dets: col = _color_for(d.label) x1, y1, x2, y2 = d.box draw.rectangle([x1, y1, x2, y2], outline=col, width=thick) text = f"{d.label} {d.conf:.2f}" try: tb = draw.textbbox((x1, y1), text, font=font) tw, th = tb[2] - tb[0], tb[3] - tb[1] except Exception: # noqa: BLE001 tw, th = len(text) * 7, 12 ty = max(0, y1 - th - 4) draw.rectangle([x1, ty, x1 + tw + 6, ty + th + 4], fill=col) draw.text((x1 + 3, ty + 1), text, fill=(255, 255, 255), font=font) return out def _prepare(image: Image.Image) -> Image.Image: img = image.convert("RGB") if max(img.size) > MAX_SIDE: img = img.copy() img.thumbnail((MAX_SIDE, MAX_SIDE)) return img def parse_prompts(text: str, limit: int = 12) -> Tuple[str, ...]: items = [] for part in (text or "").replace("\n", ",").split(","): p = part.strip().lower() if p and p not in items: items.append(p[:40]) return tuple(items[:limit]) # -------------------------------------------------------------------------- # Detection # -------------------------------------------------------------------------- def _run(model, img: Image.Image, conf: float, iou: float = 0.7, names_override=None) -> List[Detection]: results = model.predict(img, conf=conf, iou=iou, imgsz=640, verbose=False, device="cpu") res = results[0] names = names_override or res.names dets: List[Detection] = [] if res.boxes is None: return dets for box in res.boxes: cls_id = int(box.cls[0]) label = names[cls_id] if isinstance(names, (list, tuple)) else names.get(cls_id, str(cls_id)) x1, y1, x2, y2 = [int(round(v)) for v in box.xyxy[0].tolist()] dets.append(Detection(label=str(label), conf=float(box.conf[0]), box=(x1, y1, x2, y2))) dets.sort(key=lambda d: -d.conf) return dets def detect_real(image: Image.Image, mode: str = "standard", model_choice: str = "YOLOv8n (fastest)", prompts_text: str = DEFAULT_PROMPTS, conf: float = 0.25, iou: float = 0.7) -> RealDetectionResult: """mode: "standard" (COCO YOLOv8) or "open" (YOLOE, text prompts). `iou` is the overlap threshold for non-max suppression. Lower values merge overlapping boxes (fewer duplicate detections of one object) but can merge genuinely separate, touching objects such as adjacent pallets -- so it is a user-facing control rather than a value tuned on a single test image.""" img = _prepare(image) notes: List[str] = [] t0 = time.perf_counter() if mode == "open": prompts = parse_prompts(prompts_text) if not prompts: raise ValueError("Enter at least one thing to look for, e.g. 'pallet, forklift'.") model = _get_open(prompts) dets = _run(model, img, conf, iou) model_name = f"YOLOE (open-vocabulary) โ€” looking for: {', '.join(prompts)}" notes.append("Open-vocabulary detection is zero-shot: no warehouse-specific training, " "and its accuracy on warehouse photos has **not** been measured here.") else: weights = MODEL_CHOICES.get(model_choice, "yolov8n.pt") model = _get_yolo(weights) dets = _run(model, img, conf, iou) model_name = f"{weights} (COCO-pretrained, 80 classes)" notes.append("COCO has no *pallet*, *forklift*, *box* or *rack* class, so this mode " "can't count them โ€” it finds people and vehicles. Switch to open-vocabulary " "mode to search for pallets by name.") ms = (time.perf_counter() - t0) * 1000 counts: Dict[str, int] = {} for d in dets: counts[d.label] = counts.get(d.label, 0) + 1 return RealDetectionResult(_annotate(img, dets), dets, counts, ms, model_name, notes) def summarize(result: RealDetectionResult) -> str: if not result.detections: body = "**No objects detected** at this confidence threshold โ€” try lowering it." else: rows = "\n".join(f"| {k} | {v} |" for k, v in sorted(result.counts.items(), key=lambda kv: -kv[1])) body = f"**{len(result.detections)} detections**\n\n| Class | Count |\n|---|---|\n{rows}" groups: Dict[str, int] = {} for lbl, n in result.counts.items(): g = WAREHOUSE_GROUPS.get(lbl) if g: groups[g] = groups.get(g, 0) + n if groups.get("people"): body += (f"\n\n๐Ÿ‘ท **{groups['people']} person/people in frame** โ€” a production system would " f"cross-check this against AGV / forklift lanes for safety alerts.") meta = f"\n\n*Model:* {result.model_name} ยท *Inference:* {result.inference_ms:.0f} ms (measured live on this server)" notes = "".join(f"\n\n> โ„น๏ธ {n}" for n in result.notes) return body + meta + notes # -------------------------------------------------------------------------- # Attributed example photos (Wikimedia Commons, fetched at runtime) # -------------------------------------------------------------------------- def attribution_md(name: str) -> str: info = EXAMPLE_PHOTOS[name] page = "https://commons.wikimedia.org/wiki/File:" + info["file"] return (f"๐Ÿ“ท *{name}* โ€” by **{info['author']}**, via [Wikimedia Commons]({page}), " f"licensed [{info['license']}]({info['license_url']}). The annotated output is a derivative " f"work, shared under the same license.") def fetch_example(name: str) -> Tuple[Optional[Image.Image], str]: """Returns (image or None, markdown message).""" if name not in EXAMPLE_PHOTOS: return None, "Pick an example from the list, or upload your own photo." info = EXAMPLE_PHOTOS[name] os.makedirs(EXAMPLES_DIR, exist_ok=True) path = os.path.join(EXAMPLES_DIR, info["file"]) try: if not (os.path.exists(path) and os.path.getsize(path) > 10_000): url = f"https://commons.wikimedia.org/wiki/Special:FilePath/{info['file']}?width=1280" r = requests.get(url, timeout=30, headers={"User-Agent": USER_AGENT}, allow_redirects=True) r.raise_for_status() if "image" not in r.headers.get("Content-Type", ""): raise ValueError(f"unexpected content type {r.headers.get('Content-Type')}") with open(path, "wb") as f: f.write(r.content) img = Image.open(path).convert("RGB") return img, attribution_md(name) except Exception as e: # noqa: BLE001 return None, (f"โš ๏ธ Couldn't download the example photo ({type(e).__name__}). " f"This server may be offline โ€” please upload your own warehouse photo instead.")