Spaces:
Sleeping
Sleeping
Download src/real_vision.py from Pro-Coder/SmartWareHouseAI: direct link, hf CLI and curl.
- Browser
- Download file 14.6 kB
-
https://huggingface.co/spaces/Pro-Coder/SmartWareHouseAI/resolve/main/src/real_vision.py
- Command line
-
hf download hf://spaces/Pro-Coder/SmartWareHouseAI/src/real_vision.py
-
curl -L -o real_vision.py https://huggingface.co/spaces/Pro-Coder/SmartWareHouseAI/resolve/main/src/real_vision.py
14.6 kB
| """ | |
| real_vision.py | |
| --------------- | |
| Real-photo detection for the "Visual Inventory Scan" tab. | |
| Two detector modes, both from the Ultralytics YOLO family: | |
| 1. **Standard (COCO-pretrained YOLOv8)** -- fast, but COCO's 80 classes do NOT | |
| include pallet, forklift, cardboard box or rack. It reliably finds people | |
| and vehicles (useful for safety checks) but cannot count pallets. The UI | |
| says so explicitly rather than letting a user assume otherwise. | |
| 2. **Open-vocabulary (YOLOE)** -- you type what to look for ("pallet, | |
| cardboard box, forklift") and it searches for those names zero-shot, no | |
| warehouse-specific training. Accuracy on warehouse imagery is NOT measured | |
| here (no labelled real warehouse photos were available), so treat counts | |
| as a starting point, not ground truth. | |
| Why YOLOE rather than YOLO-World: both are open-vocabulary YOLO models, but | |
| YOLO-World's text encoder (CLIP) is hosted on a CDN that could not be | |
| reached from the development environment, so it could not be verified end | |
| to end. YOLOE's text encoder (MobileCLIP) ships from the same GitHub | |
| releases as the weights, and it was tested on a real photo before shipping. | |
| Design notes | |
| ------------ | |
| * Weights are downloaded on first use from the official Ultralytics GitHub | |
| release assets and cached; nothing is bundled in the repo. | |
| * `ultralytics`/`torch` are imported lazily, so if they're unavailable the | |
| rest of the app still works and this tab shows a clear message. | |
| * Example photos are fetched at runtime from Wikimedia Commons (openly | |
| licensed) and shown with author + license. They are never bundled, so the | |
| repo contains no third-party images. If the download fails, the user is | |
| told to upload their own photo. | |
| * Uploaded images are processed in memory and not stored. | |
| """ | |
| import hashlib | |
| import os | |
| import tempfile | |
| import threading | |
| import time | |
| from dataclasses import dataclass, field | |
| from typing import Dict, List, Optional, Tuple | |
| import requests | |
| from PIL import Image, ImageDraw, ImageFont | |
| WEIGHTS_DIR = os.environ.get("YOLO_WEIGHTS_DIR", os.path.join(tempfile.gettempdir(), "yolo_weights")) | |
| EXAMPLES_DIR = os.path.join(tempfile.gettempdir(), "warehouse_examples") | |
| ASSETS_BASE = "https://github.com/ultralytics/assets/releases/download/v8.3.0/" | |
| USER_AGENT = "SmartWarehouseAIDemo/1.0 (portfolio demo; Hugging Face Space)" | |
| MAX_SIDE = 1600 # downscale very large uploads before inference (CPU friendliness) | |
| MODEL_CHOICES = { | |
| "YOLOv8n (fastest)": "yolov8n.pt", | |
| "YOLOv8s (more accurate)": "yolov8s.pt", | |
| } | |
| OPEN_VOCAB_WEIGHTS = "yoloe-11s-seg.pt" | |
| TEXT_ENCODER_WEIGHTS = "mobileclip_blt.ts" # ~600 MB, downloaded once on first open-vocabulary use | |
| DEFAULT_PROMPTS = "pallet, cardboard box, forklift, person, shelving rack" | |
| # COCO classes that matter in a warehouse -> a plain-language group | |
| WAREHOUSE_GROUPS = { | |
| "person": "people", | |
| "truck": "vehicles", "car": "vehicles", "bus": "vehicles", | |
| "motorcycle": "vehicles", "bicycle": "vehicles", "train": "vehicles", | |
| } | |
| # Openly-licensed real photos from Wikimedia Commons (fetched at runtime, never bundled). | |
| EXAMPLE_PHOTOS: Dict[str, dict] = { | |
| "Pallet-rack warehouse with forklift": { | |
| "file": "Modern_warehouse_with_pallet_rack_storage_system.jpg", | |
| "author": "Axisadman", "license": "CC BY-SA 3.0", | |
| "license_url": "https://creativecommons.org/licenses/by-sa/3.0/", | |
| }, | |
| "Warehouse floor: pallets and forklift driver": { | |
| "file": "Allentown_Project_042.jpg", | |
| "author": "Chrisking", "license": "CC BY-SA 3.0", | |
| "license_url": "https://creativecommons.org/licenses/by-sa/3.0/", | |
| }, | |
| "Stacked CHEP pallets": { | |
| "file": "576_CHEP_pallets.jpg", | |
| "author": "train_photos (via Flickr)", "license": "CC BY-SA 2.0", | |
| "license_url": "https://creativecommons.org/licenses/by-sa/2.0/", | |
| }, | |
| } | |
| class Detection: | |
| label: str | |
| conf: float | |
| box: Tuple[int, int, int, int] # x1, y1, x2, y2 | |
| class RealDetectionResult: | |
| annotated: Image.Image | |
| detections: List[Detection] | |
| counts: Dict[str, int] | |
| inference_ms: float | |
| model_name: str | |
| notes: List[str] = field(default_factory=list) | |
| # -------------------------------------------------------------------------- | |
| # Weights + model cache | |
| # -------------------------------------------------------------------------- | |
| _MODEL_LOCK = threading.Lock() | |
| _MODELS: Dict[str, object] = {} | |
| def _ensure_weights(filename: str) -> str: | |
| """Download an Ultralytics release asset once and cache it.""" | |
| os.makedirs(WEIGHTS_DIR, exist_ok=True) | |
| path = os.path.join(WEIGHTS_DIR, filename) | |
| if os.path.exists(path) and os.path.getsize(path) > 1_000_000: | |
| return path | |
| tmp = path + ".part" | |
| with requests.get(ASSETS_BASE + filename, stream=True, timeout=60, | |
| headers={"User-Agent": USER_AGENT}) as r: | |
| r.raise_for_status() | |
| with open(tmp, "wb") as f: | |
| for chunk in r.iter_content(chunk_size=1 << 20): | |
| f.write(chunk) | |
| os.replace(tmp, path) | |
| return path | |
| def vision_stack_available() -> Tuple[bool, str]: | |
| """Is the optional detector stack (ultralytics + torch) importable?""" | |
| try: | |
| import ultralytics # noqa: F401 | |
| return True, "" | |
| except Exception as e: # noqa: BLE001 | |
| return False, f"{type(e).__name__}: {e}" | |
| def _quiet_settings(): | |
| """Uploaded photos are user data: turn off Ultralytics' anonymous usage sync.""" | |
| try: | |
| from ultralytics import settings | |
| settings.update({"sync": False, "weights_dir": WEIGHTS_DIR}) | |
| except Exception: # noqa: BLE001 | |
| pass | |
| def _get_yolo(weights_name: str): | |
| with _MODEL_LOCK: | |
| if weights_name not in _MODELS: | |
| from ultralytics import YOLO | |
| _quiet_settings() | |
| _MODELS[weights_name] = YOLO(_ensure_weights(weights_name)) | |
| return _MODELS[weights_name] | |
| def _get_open(prompts: Tuple[str, ...]): | |
| """YOLOE with a given vocabulary. set_classes() re-encodes the text prompts, | |
| so cache one model per distinct prompt set (a few at most).""" | |
| key = "open::" + "|".join(prompts) | |
| with _MODEL_LOCK: | |
| if key not in _MODELS: | |
| from ultralytics import YOLOE | |
| _quiet_settings() | |
| # make sure the ~600 MB text encoder is fetched from the official release | |
| # assets into our cache dir, where ultralytics will look for it | |
| _ensure_weights(TEXT_ENCODER_WEIGHTS) | |
| model = YOLOE(_ensure_weights(OPEN_VOCAB_WEIGHTS)) | |
| model.set_classes(list(prompts), model.get_text_pe(list(prompts))) | |
| _MODELS[key] = model | |
| # keep the cache from growing without bound if people try many prompt sets | |
| open_keys = [k for k in _MODELS if k.startswith("open::")] | |
| for old in open_keys[:-3]: | |
| _MODELS.pop(old, None) | |
| return _MODELS[key] | |
| # -------------------------------------------------------------------------- | |
| # Drawing | |
| # -------------------------------------------------------------------------- | |
| _PALETTE = [(231, 76, 60), (52, 152, 219), (46, 204, 113), (241, 196, 15), | |
| (155, 89, 182), (230, 126, 34), (26, 188, 156), (233, 30, 99)] | |
| def _color_for(label: str) -> Tuple[int, int, int]: | |
| h = int(hashlib.md5(label.encode()).hexdigest(), 16) | |
| return _PALETTE[h % len(_PALETTE)] | |
| def _font(size: int): | |
| try: | |
| return ImageFont.load_default(size=size) # Pillow >= 10.1 | |
| except TypeError: | |
| return ImageFont.load_default() | |
| def _annotate(image: Image.Image, dets: List[Detection]) -> Image.Image: | |
| out = image.convert("RGB").copy() | |
| draw = ImageDraw.Draw(out) | |
| w, h = out.size | |
| thick = max(2, round(max(w, h) / 400)) | |
| font = _font(max(12, round(max(w, h) / 55))) | |
| for d in dets: | |
| col = _color_for(d.label) | |
| x1, y1, x2, y2 = d.box | |
| draw.rectangle([x1, y1, x2, y2], outline=col, width=thick) | |
| text = f"{d.label} {d.conf:.2f}" | |
| try: | |
| tb = draw.textbbox((x1, y1), text, font=font) | |
| tw, th = tb[2] - tb[0], tb[3] - tb[1] | |
| except Exception: # noqa: BLE001 | |
| tw, th = len(text) * 7, 12 | |
| ty = max(0, y1 - th - 4) | |
| draw.rectangle([x1, ty, x1 + tw + 6, ty + th + 4], fill=col) | |
| draw.text((x1 + 3, ty + 1), text, fill=(255, 255, 255), font=font) | |
| return out | |
| def _prepare(image: Image.Image) -> Image.Image: | |
| img = image.convert("RGB") | |
| if max(img.size) > MAX_SIDE: | |
| img = img.copy() | |
| img.thumbnail((MAX_SIDE, MAX_SIDE)) | |
| return img | |
| def parse_prompts(text: str, limit: int = 12) -> Tuple[str, ...]: | |
| items = [] | |
| for part in (text or "").replace("\n", ",").split(","): | |
| p = part.strip().lower() | |
| if p and p not in items: | |
| items.append(p[:40]) | |
| return tuple(items[:limit]) | |
| # -------------------------------------------------------------------------- | |
| # Detection | |
| # -------------------------------------------------------------------------- | |
| def _run(model, img: Image.Image, conf: float, iou: float = 0.7, names_override=None) -> List[Detection]: | |
| results = model.predict(img, conf=conf, iou=iou, imgsz=640, verbose=False, device="cpu") | |
| res = results[0] | |
| names = names_override or res.names | |
| dets: List[Detection] = [] | |
| if res.boxes is None: | |
| return dets | |
| for box in res.boxes: | |
| cls_id = int(box.cls[0]) | |
| label = names[cls_id] if isinstance(names, (list, tuple)) else names.get(cls_id, str(cls_id)) | |
| x1, y1, x2, y2 = [int(round(v)) for v in box.xyxy[0].tolist()] | |
| dets.append(Detection(label=str(label), conf=float(box.conf[0]), box=(x1, y1, x2, y2))) | |
| dets.sort(key=lambda d: -d.conf) | |
| return dets | |
| def detect_real(image: Image.Image, mode: str = "standard", model_choice: str = "YOLOv8n (fastest)", | |
| prompts_text: str = DEFAULT_PROMPTS, conf: float = 0.25, iou: float = 0.7) -> RealDetectionResult: | |
| """mode: "standard" (COCO YOLOv8) or "open" (YOLOE, text prompts). | |
| `iou` is the overlap threshold for non-max suppression. Lower values merge | |
| overlapping boxes (fewer duplicate detections of one object) but can merge | |
| genuinely separate, touching objects such as adjacent pallets -- so it is a | |
| user-facing control rather than a value tuned on a single test image.""" | |
| img = _prepare(image) | |
| notes: List[str] = [] | |
| t0 = time.perf_counter() | |
| if mode == "open": | |
| prompts = parse_prompts(prompts_text) | |
| if not prompts: | |
| raise ValueError("Enter at least one thing to look for, e.g. 'pallet, forklift'.") | |
| model = _get_open(prompts) | |
| dets = _run(model, img, conf, iou) | |
| model_name = f"YOLOE (open-vocabulary) — looking for: {', '.join(prompts)}" | |
| notes.append("Open-vocabulary detection is zero-shot: no warehouse-specific training, " | |
| "and its accuracy on warehouse photos has **not** been measured here.") | |
| else: | |
| weights = MODEL_CHOICES.get(model_choice, "yolov8n.pt") | |
| model = _get_yolo(weights) | |
| dets = _run(model, img, conf, iou) | |
| model_name = f"{weights} (COCO-pretrained, 80 classes)" | |
| notes.append("COCO has no *pallet*, *forklift*, *box* or *rack* class, so this mode " | |
| "can't count them — it finds people and vehicles. Switch to open-vocabulary " | |
| "mode to search for pallets by name.") | |
| ms = (time.perf_counter() - t0) * 1000 | |
| counts: Dict[str, int] = {} | |
| for d in dets: | |
| counts[d.label] = counts.get(d.label, 0) + 1 | |
| return RealDetectionResult(_annotate(img, dets), dets, counts, ms, model_name, notes) | |
| def summarize(result: RealDetectionResult) -> str: | |
| if not result.detections: | |
| body = "**No objects detected** at this confidence threshold — try lowering it." | |
| else: | |
| rows = "\n".join(f"| {k} | {v} |" for k, v in sorted(result.counts.items(), key=lambda kv: -kv[1])) | |
| body = f"**{len(result.detections)} detections**\n\n| Class | Count |\n|---|---|\n{rows}" | |
| groups: Dict[str, int] = {} | |
| for lbl, n in result.counts.items(): | |
| g = WAREHOUSE_GROUPS.get(lbl) | |
| if g: | |
| groups[g] = groups.get(g, 0) + n | |
| if groups.get("people"): | |
| body += (f"\n\n👷 **{groups['people']} person/people in frame** — a production system would " | |
| f"cross-check this against AGV / forklift lanes for safety alerts.") | |
| meta = f"\n\n*Model:* {result.model_name} · *Inference:* {result.inference_ms:.0f} ms (measured live on this server)" | |
| notes = "".join(f"\n\n> ℹ️ {n}" for n in result.notes) | |
| return body + meta + notes | |
| # -------------------------------------------------------------------------- | |
| # Attributed example photos (Wikimedia Commons, fetched at runtime) | |
| # -------------------------------------------------------------------------- | |
| def attribution_md(name: str) -> str: | |
| info = EXAMPLE_PHOTOS[name] | |
| page = "https://commons.wikimedia.org/wiki/File:" + info["file"] | |
| return (f"📷 *{name}* — by **{info['author']}**, via [Wikimedia Commons]({page}), " | |
| f"licensed [{info['license']}]({info['license_url']}). The annotated output is a derivative " | |
| f"work, shared under the same license.") | |
| def fetch_example(name: str) -> Tuple[Optional[Image.Image], str]: | |
| """Returns (image or None, markdown message).""" | |
| if name not in EXAMPLE_PHOTOS: | |
| return None, "Pick an example from the list, or upload your own photo." | |
| info = EXAMPLE_PHOTOS[name] | |
| os.makedirs(EXAMPLES_DIR, exist_ok=True) | |
| path = os.path.join(EXAMPLES_DIR, info["file"]) | |
| try: | |
| if not (os.path.exists(path) and os.path.getsize(path) > 10_000): | |
| url = f"https://commons.wikimedia.org/wiki/Special:FilePath/{info['file']}?width=1280" | |
| r = requests.get(url, timeout=30, headers={"User-Agent": USER_AGENT}, allow_redirects=True) | |
| r.raise_for_status() | |
| if "image" not in r.headers.get("Content-Type", ""): | |
| raise ValueError(f"unexpected content type {r.headers.get('Content-Type')}") | |
| with open(path, "wb") as f: | |
| f.write(r.content) | |
| img = Image.open(path).convert("RGB") | |
| return img, attribution_md(name) | |
| except Exception as e: # noqa: BLE001 | |
| return None, (f"⚠️ Couldn't download the example photo ({type(e).__name__}). " | |
| f"This server may be offline — please upload your own warehouse photo instead.") | |