Spaces:
Running on Zero
Running on Zero
Download app.py from Pro-Coder/SmartWareHouseAI: direct link, hf CLI and curl.
- Browser
- Download file 64.6 kB
-
https://huggingface.co/spaces/Pro-Coder/SmartWareHouseAI/resolve/main/app.py
- Command line
-
hf download hf://spaces/Pro-Coder/SmartWareHouseAI/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Pro-Coder/SmartWareHouseAI/resolve/main/app.py
64.6 kB
"""
Smart Warehouse AI Assistant
=============================
A warehouse/intralogistics AI copilot demo, built for a Hugging Face
Space. Combines:
1. An LLM-powered assistant (RAG: TF-IDF retrieval + hosted LLM via the
an OpenAI-compatible API) for natural-language warehouse operations Q&A.
2. A TF-IDF + Logistic Regression intent classifier that routes queries
into 8 operational categories.
3. A lightweight NL -> structured query layer over synthetic inventory /
order tables.
4. An Isolation Forest anomaly detector for conveyor/crane sensor
streams (predictive maintenance).
5. A Model Evaluation tab reporting real accuracy/F1/ROC-AUC metrics
computed by build_artifacts.py.
Author: (your name here) -- built as a portfolio / job-application project.
"""
import json
import os
import gradio as gr
import pandas as pd
from src.anomaly_model import FEATURES as SENSOR_FEATURES
from src.anomaly_model import load_artifacts as load_anomaly_artifacts
from src.anomaly_model import score_reading
from src.alerting import send_anomaly_alert
from src.vision_model import generate_rack_scene, detect_boxes
from src.real_vision import (
DEFAULT_PROMPTS, EXAMPLE_PHOTOS, MODEL_CHOICES, detect_real, fetch_example,
summarize as summarize_real, vision_stack_available,
)
from src.data_generation import generate_inventory_db, generate_orders_db
from src.intent_model import INTENT_DESCRIPTIONS, load_pipeline, predict as intent_predict
from src.inventory_db import query_inventory, query_orders
from src.llm_client import answer_query, test_connection, llm_configured
from src.retriever import KBRetriever
ROOT = os.path.dirname(os.path.abspath(__file__))
MODELS_DIR = os.path.join(ROOT, "models")
DATA_DIR = os.path.join(ROOT, "data")
ASSETS_DIR = os.path.join(ROOT, "assets")
# --------------------------------------------------------------------------
# Load pre-trained artifacts (fast: no training happens at Space startup
# under normal conditions). If unpickling fails -- e.g. the models were
# built with a different scikit-learn version than the one installed in
# this container -- we transparently rebuild everything from scratch. This
# is fast (a few seconds, see build_artifacts.py) and makes the app immune
# to sklearn version-pinning mismatches between build time and deploy time.
# --------------------------------------------------------------------------
def _load_or_rebuild_artifacts():
intent_path = os.path.join(MODELS_DIR, "intent_pipeline.joblib")
anomaly_model_path = os.path.join(MODELS_DIR, "anomaly_iforest.joblib")
anomaly_scaler_path = os.path.join(MODELS_DIR, "anomaly_scaler.joblib")
anomaly_threshold_path = os.path.join(MODELS_DIR, "anomaly_threshold.joblib")
def _try_load():
pipeline = load_pipeline(intent_path)
model, scaler, threshold = load_anomaly_artifacts(anomaly_model_path, anomaly_scaler_path, threshold_path=anomaly_threshold_path)
# Smoke-test the loaded pipeline against the exact code path the app
# uses at request time. On a scikit-learn version mismatch, sklearn
# sometimes unpickles "successfully" but throws later on first real
# use (e.g. LogisticRegression missing an internal attribute) --
# catching that here, at import time, is what makes this self-healing.
intent_predict(pipeline, "healthcheck")
score_reading(model, scaler, {"motor_temp_c": 55, "vibration_mm_s": 2.2, "current_amps": 12, "belt_speed_mps": 1.5}, threshold=threshold)
return pipeline, model, scaler, threshold
try:
return _try_load()
except Exception as e: # noqa: BLE001 -- any load/version issue triggers a rebuild
print(f"[startup] Could not load pre-built model artifacts ({type(e).__name__}: {e}). "
f"Rebuilding from scratch with the installed scikit-learn version...")
import build_artifacts
build_artifacts.build_intent_classifier()
build_artifacts.build_anomaly_detector()
build_artifacts.build_retrieval_eval()
build_artifacts.build_inventory_and_orders()
print("[startup] Rebuild complete.")
return _try_load()
intent_pipeline, anomaly_model, anomaly_scaler, anomaly_threshold = _load_or_rebuild_artifacts()
retriever = KBRetriever()
# Prefer the pre-generated CSVs (so demo state matches the eval run); fall back to
# regenerating in-memory if they're missing for some reason.
try:
inventory_df = pd.read_csv(os.path.join(DATA_DIR, "inventory.csv"))
orders_df = pd.read_csv(os.path.join(DATA_DIR, "orders.csv"))
except FileNotFoundError:
inventory_df = generate_inventory_db()
orders_df = generate_orders_db()
def _load_json(name):
path = os.path.join(DATA_DIR, name)
if os.path.exists(path):
with open(path) as f:
return json.load(f)
return {}
intent_eval = _load_json("intent_eval.json")
anomaly_eval = _load_json("anomaly_eval.json")
retrieval_eval = _load_json("retrieval_eval.json")
latency_eval = _load_json("latency_eval.json")
vision_eval = _load_json("vision_eval.json")
LLM_CONFIGURED = llm_configured()
# --------------------------------------------------------------------------
# ZeroGPU compatibility shim
# --------------------------------------------------------------------------
# This app is CPU-only by design (scikit-learn locally, LLM calls go to the
# remote LLM API). Some Spaces accounts, however, only offer the
# free "ZeroGPU" hardware tier, which requires at least one function
# decorated with `@spaces.GPU` to be present or the platform's startup
# check fails with "No @spaces.GPU function detected". This is a harmless,
# unused health-check function that satisfies that requirement without
# changing any real behaviour -- it is never called on the request path.
try:
import spaces
@spaces.GPU(duration=5)
def _zerogpu_healthcheck():
return True
except ImportError:
# Running locally / on CPU-basic hardware where `spaces` isn't installed.
def _zerogpu_healthcheck():
return True
# ==========================================================================
# TAB 1 -- AI Assistant (LLM + RAG + intent routing)
# ==========================================================================
def chat_fn(message, history):
intent = intent_predict(intent_pipeline, message)
# `history` (Gradio messages format) is passed through so the LLM can handle
# follow-ups like "what about Zone D instead?" (bounded/cleaned in llm_client).
response = answer_query(message, retriever, history=history)
intent_label = INTENT_DESCRIPTIONS.get(intent.intent, intent.intent)
meta_lines = [f"**Detected intent:** {intent_label} ({intent.confidence:.0%} confidence)"]
if response.sources:
src_str = ", ".join(f"{s.title} ({s.score:.2f})" for s in response.sources)
meta_lines.append(f"**Retrieved context:** {src_str}")
meta_lines.append(
f"**Generation:** {'LLM (' + response.model_id + ')' if response.used_llm else 'retrieval-only fallback'}"
f" · {response.latency_s * 1000:.0f} ms"
)
if not response.used_llm and response.debug_errors:
err_lines = "\n".join(f" - `{e}`" for e in response.debug_errors)
meta_lines.append(f"**Why the LLM wasn't used:**\n{err_lines}")
full_reply = response.answer + "\n\n---\n" + "\n".join(meta_lines)
return full_reply
def test_llm_fn():
response = test_connection(retriever)
if response.used_llm:
return (
f"✅ **LLM connection working.** Model: `{response.model_id}` · "
f"{response.latency_s * 1000:.0f} ms\n\nSample answer: {response.answer}"
)
err_lines = "\n".join(f"- {e}" for e in response.debug_errors) or "(no error detail captured)"
return (
"❌ **LLM connection failed** — running in retrieval-only fallback mode.\n\n"
f"**What each configured backend reported:**\n{err_lines}\n\n"
"**Most reliable fix:** add a `GROQ_API_KEY` Space secret (free key at console.groq.com), then "
"restart the Space. Hugging Face's router needs inference credits, and free accounts get almost none "
"(that's the `402 Payment Required` error). Any OpenAI-compatible API also works via "
"`LLM_API_KEY` + `LLM_BASE_URL` + `LLM_MODEL`."
)
ASSISTANT_EXAMPLES = [
"The conveyor belt in Zone C is making noise",
"How many units of SKU-1042 are in Zone B?",
"What's the status of order #10007?",
"What's the difference between an AGV and an AMR?",
"A forklift near-miss was reported in Zone A",
"What's the fastest picking route for a high volume order?",
"Is Crane-03 operational?",
"Route AGV-12 to picking station 5",
"What is cycle counting?",
"Redirect AGV-07 around the blocked aisle in Zone B",
"How does an AS/RS crane retrieve a pallet?",
"What KPIs matter most in warehouse automation?",
"Schedule maintenance for Sorter-02",
"Should we batch pick these orders together?",
"What is predictive maintenance?",
"Log a safety incident involving AMR-21",
]
# ==========================================================================
# TAB 2 -- Inventory & Order Query
# ==========================================================================
def inventory_query_fn(text):
intent = intent_predict(intent_pipeline, text)
if intent.intent == "order_status":
result = query_orders(orders_df, text)
note = "Interpreted as an **order status** query."
else:
result = query_inventory(inventory_df, text)
note = "Interpreted as an **inventory** query."
if result.empty:
result = pd.DataFrame({"message": ["No matching records found for this query."]})
return note, result
# ==========================================================================
# TAB 3 -- Predictive Maintenance / Anomaly Detection
# ==========================================================================
def anomaly_fn(motor_temp, vibration, current, belt_speed, zone_label="", send_alert=False):
reading = {
"motor_temp_c": motor_temp, "vibration_mm_s": vibration,
"current_amps": current, "belt_speed_mps": belt_speed,
}
result = score_reading(anomaly_model, anomaly_scaler, reading, threshold=anomaly_threshold)
verdict = "🔴 ANOMALY DETECTED" if result.is_anomaly else "🟢 Normal operating range"
detail = (
f"### {verdict}\n\n"
f"**Anomaly score:** {result.anomaly_score:.2f} / 1.00\n\n"
f"| Sensor | Value | Typical normal range |\n|---|---|---|\n"
f"| Motor temperature | {motor_temp:.1f} °C | 47–63 °C |\n"
f"| Vibration | {vibration:.2f} mm/s | 1.2–3.2 mm/s |\n"
f"| Motor current | {current:.1f} A | 9–15 A |\n"
f"| Belt speed | {belt_speed:.2f} m/s | 1.2–1.8 m/s |\n"
)
if result.is_anomaly:
detail += (
"\n**Recommended action:** Flag for inspection. Elevated temperature + "
"vibration + current with reduced belt speed typically indicates bearing "
"wear, belt misalignment, or motor overload -- schedule maintenance before "
"the next shift to avoid an unplanned stoppage."
)
if send_alert:
alert = send_anomaly_alert(zone_label or "Unspecified zone", reading, result.anomaly_score)
if alert.sent:
detail += "\n\n---\n✅ **Alert sent to Slack.**"
else:
why = alert.error or "no `SLACK_WEBHOOK_URL` secret configured"
detail += (f"\n\n---\n📋 **Alert preview (NOT actually sent — {why}):**\n"
f"```\n{alert.message_preview}\n```")
return detail
ANOMALY_PRESETS = {
"Normal reading": (55.0, 2.2, 12.0, 1.5),
"Early bearing wear": (68.0, 3.8, 15.5, 1.3),
"Severe fault (imminent failure)": (85.0, 6.5, 21.0, 0.6),
}
def load_preset(name):
return ANOMALY_PRESETS[name]
def anomaly_batch_fn(csv_file):
"""CSV batch mode: run anomaly detection over many readings at once
(e.g. an export of the last shift's sensor logs)."""
if csv_file is None:
return "Upload a CSV first.", None, None
try:
batch_df = pd.read_csv(csv_file.name if hasattr(csv_file, "name") else csv_file)
except Exception as e: # noqa: BLE001
return f"Could not read CSV: {type(e).__name__}: {e}", None, None
missing = [c for c in SENSOR_FEATURES if c not in batch_df.columns]
if missing:
return (f"CSV is missing column(s): {', '.join(missing)}. Expected: "
f"{', '.join(SENSOR_FEATURES)} (optionally plus an `id` column).", None, None)
rows = []
for _, row in batch_df.iterrows():
reading = {f: float(row[f]) for f in SENSOR_FEATURES}
r = score_reading(anomaly_model, anomaly_scaler, reading, threshold=anomaly_threshold)
rows.append({**({"id": row["id"]} if "id" in batch_df.columns else {}),
**reading, "anomaly_score": round(r.anomaly_score, 3), "is_anomaly": r.is_anomaly})
out = pd.DataFrame(rows)
n_bad = int(out["is_anomaly"].sum())
summary = f"**Processed {len(out)} readings** — {n_bad} flagged anomalous ({n_bad / max(len(out), 1):.1%})."
path = os.path.join("/tmp", "anomaly_batch_results.csv")
out.to_csv(path, index=False)
return summary, out, path
SAMPLE_BATCH_CSV = os.path.join(DATA_DIR, "sample_sensor_batch.csv")
if not os.path.exists(SAMPLE_BATCH_CSV): # self-heal if the packaged sample is ever missing
import build_artifacts
build_artifacts.build_sample_batch_csv()
# ==========================================================================
# TAB 4 -- Visual Inventory Scan (classical computer vision)
# ==========================================================================
def vision_scan_fn(fill_density, noise_level, seed_state):
seed = int(seed_state) if seed_state is not None else 1
scene = generate_rack_scene(fill_prob=fill_density, seed=seed, noise_level=noise_level)
result = detect_boxes(scene.image)
diff = abs(result.detected_count - scene.true_box_count)
verdict = "✅ Exact match" if diff == 0 else f"⚠️ Off by {diff}"
summary = (
f"### Detected **{result.detected_count} boxes** · estimated occupancy **{result.estimated_occupancy:.0%}**\n\n"
f"| | CV pipeline | Ground truth |\n|---|---|---|\n"
f"| Box count | {result.detected_count} | {scene.true_box_count} |\n"
f"| Occupancy | {result.estimated_occupancy:.0%} | {scene.true_occupancy:.0%} |\n\n"
f"**{verdict}** — ground truth exists only because this scene is synthetic; "
f"a real deployment wouldn't have it."
)
return scene.image, result.annotated_image, summary, seed + 1
def real_detect_fn(image, mode, model_choice, prompts_text, conf, iou=0.7):
"""Run a real-photo detector (YOLO) on an uploaded / example image."""
if image is None:
return None, "⬆️ Upload a photo, or load an example, first."
ok, why = vision_stack_available()
if not ok:
return None, (f"⚠️ The detector stack (ultralytics + PyTorch) isn't available on this server "
f"(`{why}`). The rest of the app is unaffected.")
try:
res = detect_real(image, mode=mode, model_choice=model_choice, prompts_text=prompts_text, conf=conf, iou=iou)
except ValueError as e: # bad user input (e.g. empty prompt box) -> plain guidance
return None, f"✏️ {e}"
except Exception as e: # noqa: BLE001
return None, (f"⚠️ Detection failed: `{type(e).__name__}: {e}`\n\n"
f"The first run downloads model weights from GitHub, so a network problem there "
f"is the most common cause — try again, or switch detector mode.")
return res.annotated, summarize_real(res)
def load_example_fn(name):
img, msg = fetch_example(name)
return img, msg
# ==========================================================================
# TAB 4 -- Model Evaluation
# ==========================================================================
def eval_intent_section_md():
return (
f"### 1. Intent Classifier "
f"🎯 **{intent_eval.get('accuracy', 0):.0%} accuracy** · "
f"**{intent_eval.get('macro_f1', 0):.0%} macro F1** · "
f"held-out test set, {intent_eval.get('n_test', '?')} examples, "
f"{intent_eval.get('n_classes', '?')} classes"
)
def eval_anomaly_section_md():
return (
f"### 2. Predictive Maintenance (Anomaly Detector) "
f"🎯 **{anomaly_eval.get('f1', 0):.0%} F1** · "
f"**{anomaly_eval.get('roc_auc', 0):.2f} ROC-AUC** · "
f"**{anomaly_eval.get('precision', 0):.0%} precision** · "
f"**{anomaly_eval.get('recall', 0):.0%} recall**"
)
def eval_retrieval_section_md():
return (
f"### 3. RAG Retrieval "
f"🎯 **{retrieval_eval.get('hit_rate_at_1', 0):.0%} hit-rate@1** · "
f"**{retrieval_eval.get('hit_rate_at_2', 0):.0%} hit-rate@2** · "
f"{retrieval_eval.get('n_queries', '?')} labelled test queries"
)
def eval_latency_section_md():
return (
f"### 4. Latency (CPU, avg of 50 runs) "
f"🎯 **{latency_eval.get('intent_classifier_ms', '?')} ms** intent · "
f"**{latency_eval.get('anomaly_detector_ms', '?')} ms** anomaly · "
f"**{latency_eval.get('kb_retrieval_ms', '?')} ms** retrieval"
)
EVAL_METHODOLOGY_MD = (
"*All metrics above are computed on held-out synthetic test data by "
"`build_artifacts.py` (fully reproducible) — see the **About** tab for "
"dataset details and how each model works. Two safeguards keep these "
"numbers honest: (1) train/test splits are deduplicated on exact text "
"with an automated zero-overlap check, so the intent classifier can't "
"memorize verbatim test examples; (2) the anomaly detector is fit "
"unsupervised on a training split, then its decision threshold is "
"calibrated on a separate small *labelled calibration split* (analogous "
"to a handful of confirmed historical incidents) — never on the test "
"set used to report the metrics above.*"
)
# ==========================================================================
# TAB 5 -- About
# ==========================================================================
ABOUT_MD = f"""
# 🏭 Smart Warehouse AI Assistant
**A portfolio project demonstrating an applied-AI approach to intralogistics
and warehouse automation operations.**
Modern warehouse automation platforms — automated storage & retrieval
systems (AS/RS), conveyor & sortation lines, AGVs/AMRs, and the WMS/WCS
software that orchestrates them — generate huge volumes of operational
data: equipment telemetry, transactions, safety logs, and ad-hoc questions
from floor staff. This project is a compact, end-to-end example of how an
AI layer can sit on top of that kind of system: routing requests correctly,
answering from real operational context instead of guessing, catching
equipment problems early, and doing all of it with **honest, reproducible
evaluation** rather than a demo that just "looks like it works."
## What this demonstrates
| Capability | Where | Techniques used |
|---|---|---|
| **LLM-powered natural-language assistant**, grounded with retrieval (RAG) so it answers from real domain knowledge rather than hallucinating | *AI Assistant* tab | TF-IDF retrieval + hosted LLM (HF Inference API) |
| **Intent classification** to route free-text requests (maintenance, safety, navigation, inventory, etc.) the way a real ops system would triage tickets | *AI Assistant* / *Inventory* tabs | TF-IDF + Logistic Regression |
| **Predictive maintenance** via unsupervised anomaly detection on conveyor/crane sensor streams — catching bearing wear or misalignment before an unplanned stoppage | *Predictive Maintenance* tab | Isolation Forest, unsupervised |
| **NL-to-structured-query** over inventory/order data, a lightweight stand-in for a WMS query tool | *Inventory & Order Query* tab | Regex slot extraction + intent routing |
| **Rigorous, reproducible evaluation** of every ML component (accuracy, F1, ROC-AUC, retrieval hit-rate, latency) rather than just a demo that "looks like it works" | *Model Evaluation* tab | scikit-learn metrics, held-out test splits |
## Dataset overview — what the data is and where it's from
**Every dataset in this project is synthetically generated by the project's
own code** (`src/data_generation.py` and `src/knowledge_base.py`), not
scraped, exported, or sourced from any real company's systems. This was a
deliberate choice: it keeps the project fully self-contained, reproducible,
and shareable without any data-privacy or licensing concerns, while still
being realistic enough to demonstrate the underlying ML techniques properly.
| Dataset | What it is | Size | How it's generated |
|---|---|---|---|
| **Knowledge base** | Original, hand-written articles on generic warehouse-automation concepts (AS/RS, AGV/AMR, WMS, sortation, picking strategy, predictive maintenance, safety, inventory accuracy, KPIs, energy efficiency, cross-docking, returns, RFID, dock scheduling, slotting, security, labor management) | 17 articles | Written by the developer specifically for this project |
| **Intent queries** | Example free-text operational questions/requests across 8 categories | ~480 examples | ~8 templates per category with randomised SKU codes, zone names, order IDs, and equipment IDs slotted in |
| **Inventory table** | SKU records with category, zone, on-hand units, reorder point, unit cost | 60 SKUs | Randomised within realistic ranges (fixed seed) |
| **Orders table** | Order records with status, line count, priority, zone | 80 orders | Randomised within realistic ranges (fixed seed) |
| **Sensor readings** | Conveyor/crane motor telemetry: temperature, vibration, current, belt speed | 1,000 readings (900 normal + 100 anomalous) | Normal readings drawn from realistic operating ranges; anomalies simulate known failure signatures (elevated temp/vibration/current + reduced belt speed) |
| **Retrieval eval set** | Hand-labelled (question → expected knowledge-base article) pairs | 17 pairs | Written by the developer to check retrieval accuracy |
## How each model works (function)
| Model | Purpose | Algorithm | Input → Output |
|---|---|---|---|
| **Intent classifier** | Decide what kind of request a query is (maintenance, safety, inventory, order status, navigation, picking, system status, general FAQ) | TF-IDF + Logistic Regression | Free text → intent label + confidence |
| **KB retriever** | Find the most relevant knowledge-base passage(s) for a query, to ground the LLM's answer | TF-IDF + cosine similarity | Free text → top-k ranked passages |
| **LLM assistant** | Generate a natural-language answer grounded in the retrieved context | Hosted instruct LLM via an OpenAI-compatible API (Groq / custom / HF router), with fallback | Query + context → grounded answer (or an extractive fallback if the LLM is unavailable) |
| **Inventory/order query** | Turn a question into a filtered table lookup | Regex slot extraction (SKU / order ID / zone) + intent routing | Free text → filtered inventory or orders table |
| **Anomaly detector** | Flag abnormal equipment sensor readings before they cause a stoppage | Isolation Forest (unsupervised), StandardScaler | 4 sensor features → anomaly / normal + anomaly score |
## How to test this project
1. **AI Assistant** — try one of the example questions, or ask your own
(e.g. *"The conveyor belt in Zone C is making noise"*). Check the
metadata line under each answer to see the detected intent, which
knowledge-base article(s) were retrieved, and whether the LLM or the
fallback path answered. If the LLM path isn't working, open the
**LLM connection diagnostics** accordion and click "Test LLM connection"
for a precise error message.
2. **Inventory & Order Query** — try *"How many units of SKU-1042 are in
Zone B?"* or *"What's the status of order #10007?"*, or browse the full
synthetic tables in the accordion below the search box.
3. **Predictive Maintenance** — load one of the presets (Normal / Early
bearing wear / Severe fault) or drag the sliders yourself, then click
*"Check for anomaly"* to see the model's verdict and recommended action.
4. **Model Evaluation** — every chart here is generated on **held-out test
data** by `build_artifacts.py`, not cherry-picked from a live run. Run
that script yourself to reproduce every number from scratch.
## Tech stack
- **UI / deployment:** [Gradio](https://gradio.app) on Hugging Face Spaces
- **LLM:** any hosted model behind an **OpenAI-compatible chat API** — Groq's
free tier (`GROQ_API_KEY`), a custom endpoint (`LLM_API_KEY` / `LLM_BASE_URL` /
`LLM_MODEL`), or Hugging Face's inference router. Plain HTTP rather than a
provider-specific SDK, so there's no provider list to go stale. It stops after
one attempt on a 402/401 (every model would fail the same way) and falls back
to a retrieval-only answer, so the public demo never just breaks.
- **Retrieval:** TF-IDF + cosine similarity over a small hand-written
warehouse-operations knowledge base (simple, fast, fully local RAG).
- **Intent classification:** TF-IDF + Logistic Regression (scikit-learn) --
chosen deliberately over a heavier transformer classifier because it
trains in under a second and comfortably reaches **{intent_eval.get('accuracy', 0):.0%}
accuracy** on this task; right-sizing the model to the problem.
- **Anomaly detection:** Isolation Forest (scikit-learn), trained
unsupervised on scaled sensor features.
- **Evaluation:** scikit-learn metrics + matplotlib, all computed by
`build_artifacts.py` and saved as static artifacts the app loads at
startup (fast, reproducible Space boot, self-healing if the deployed
scikit-learn version ever drifts from the one used to train the models).
## Architecture
```
┌───────────────────────┐
User query ───► │ Intent Classifier │ (TF-IDF + LogisticRegression)
└──────────┬────────────┘
│ intent label
▼
┌───────────────────────┐
│ KB Retriever (RAG) │ (TF-IDF cosine similarity)
└──────────┬────────────┘
│ top-k passages
▼
┌───────────────────────┐
│ Hosted LLM │ (OpenAI-compatible API)
│ (or extractive │
│ fallback if offline) │
└──────────┬────────────┘
▼
Grounded answer
Sensor stream ───► StandardScaler ───► IsolationForest ───► anomaly / normal
```
## Why this approach
The value of AI in a warehouse-automation context isn't a flashy chatbot —
it's **routing, grounding, and reliability**: correctly triaging a request,
answering from real operational context instead of guessing, and flagging
equipment problems before they cause downtime. Every component here was
chosen to be as simple as it can be while still doing that job well and
being honestly evaluated, rather than reaching for the biggest available
model by default.
## Extended capabilities
- **Multi-turn conversation** — the assistant keeps the last few turns, so follow-ups like *"what about Zone D instead?"* work.
- **📷 Visual Inventory Scan** — runs YOLO on **real warehouse photos** (upload your own, or load an openly-licensed example). *Standard* mode (COCO-pretrained YOLOv8) finds people and vehicles but has no pallet/forklift/box class; *Find by name* mode (open-vocabulary YOLOE) searches for whatever you type, zero-shot. **Honest limit:** accuracy on real warehouse photos is *not measured* (no labelled warehouse images were available). A classical-CV baseline on synthetic racks, where ground truth is known, *is* measured: 100% on clean renders, degrading under simulated noise (see Model Evaluation).
**Detector options considered for real photos** (latency = rough figures from a development CPU, after model load):
| Option | Finds pallets / forklifts / boxes? | Weight & speed | Status |
|---|---|---|---|
| YOLOv8 (COCO) | **No** — no such classes; finds people & vehicles | 6–22 MB, ~75 ms | ✅ Shipped; run on a real photo |
| YOLOE (open-vocabulary) | Yes, zero-shot via text prompts | 28 MB + ~600 MB text encoder, ~250 ms; first load ≈ 1–2 min | ✅ Shipped; run on a real photo (not on warehouse photos) |
| YOLO-World (open-vocabulary) | Yes, zero-shot | text encoder hosted on a CDN I couldn't reach | ❌ Not shipped — couldn't verify it |
| OWLv2 / Grounding DINO | Yes, zero-shot, often stronger | hundreds of MB, seconds per image on CPU; weights on the Hugging Face Hub, unreachable from my environment | ❌ Not shipped — couldn't verify it |
| YOLO fine-tuned on a warehouse dataset | Best accuracy | needs licensed, labelled warehouse photos + training | ⏳ Recommended next step |
| Classical CV (OpenCV) | Only clean synthetic racks | tiny | ✅ Kept as a *measured* baseline |
- **Batch CSV scoring** — upload a CSV of sensor readings (a sample is provided) to score a whole shift's logs at once, and download the results.
- **Slack alerting** — anomalies can trigger a Slack alert. Real if a `SLACK_WEBHOOK_URL` secret is set; otherwise a clearly-labelled preview that is *never* presented as sent.
- **Bilingual UI** — switch the explanatory text between English and 日本語 with the selector at the top. (Charts and metric tables stay in English; the assistant answers in whatever language you ask in.)
## Limitations & next steps
- All data here is **synthetic**, for portfolio/demo purposes — a production
version would connect to real WMS/WCS APIs and historical sensor logs.
- The intent set (8 classes) and knowledge base (17 articles) are intentionally
small to keep the demo fast and auditable; both are easy to extend.
- The anomaly detector uses 4 hand-picked features; a production system would
likely use a richer multivariate sensor set and a supervised or
semi-supervised model once labelled failure data is available.
---
*Built as a portfolio/application project. Source code available on request
or in the linked repository. Feedback welcome.*
"""
# ==========================================================================
# GRADIO APP
# ==========================================================================
CUSTOM_CSS = """
#title-banner { text-align: center; margin-bottom: 0.5em; }
.gradio-container { max-width: 1150px !important; margin: auto; }
.data-badge {
background: #eff6ff;
border-left: 4px solid #3b82f6;
border-radius: 6px;
padding: 10px 14px;
margin-bottom: 10px;
font-size: 0.92em;
line-height: 1.5;
}
.data-badge b { color: #1e3a8a; }
"""
# ==========================================================================
# Bilingual (English / 日本語) static content
# ==========================================================================
# Static explanatory text (subtitle, per-tab data/function panels, About tab)
# is switchable via the language selector. Dynamic content (charts, metric
# tables, the LLM's answers) stays in English; the assistant itself will
# answer in whatever language the question is asked in, since the hosted LLM
# is multilingual.
from src.knowledge_base import KNOWLEDGE_BASE
_N_KB = len(KNOWLEDGE_BASE)
I18N = {
"subtitle": {
"en": "<p style='text-align:center; color:gray;'>LLM-powered intralogistics copilot · intent routing · "
"multi-turn RAG · predictive maintenance · computer vision</p>",
"ja": "<p style='text-align:center; color:gray;'>LLM搭載の物流オペレーション支援AI · 意図分類 · "
"マルチターン対話(RAG) · 予知保全 · コンピュータビジョン</p>",
},
"badge_assistant": {
"en": "🧪 <b>Source:</b> hand-written by the developer — %d original knowledge-base articles on warehouse "
"operations (AS/RS, AGV/AMR, WMS, sortation, picking, maintenance, safety, cross-docking, RFID, slotting, "
"labor management…) plus ~480 template-generated example queries. Not scraped or sourced from any real company.<br>"
"<b>What it describes:</b> generic intralogistics/automation concepts and operational scenarios (equipment "
"faults, safety incidents, order/inventory questions) — general domain knowledge, not any specific facility's live data.<br>"
"<b>Function:</b> answers free-text warehouse-ops questions by classifying the query's intent, retrieving the "
"most relevant knowledge-base passage(s) (RAG), then generating a grounded answer with a hosted LLM. It keeps the "
"last few turns of the conversation, so follow-ups like “what about Zone D instead?” work. If no LLM is available, "
"it falls back to showing the retrieved passages directly." % _N_KB,
"ja": "🧪 <b>データの出典:</b>開発者が手書きした倉庫運用ナレッジ記事%d本(AS/RS、AGV/AMR、WMS、仕分け、ピッキング、保全、安全、"
"クロスドッキング、RFID、スロッティング、労務管理など)と、テンプレートから生成した約480件の問い合わせ例。"
"実在企業のデータは使用していません。<br>"
"<b>内容:</b>物流・自動化に関する一般的な概念と運用シナリオ(設備故障、安全インシデント、在庫・注文の問い合わせ等)であり、"
"特定施設のリアルタイムデータではありません。<br>"
"<b>機能:</b>自由入力の質問に対して、①意図分類 → ②ナレッジ検索(RAG)→ ③ホスト型LLMによる回答生成の順に処理します。"
"直近の会話履歴も保持するため、「Zone Dの場合は?」のような追加質問にも対応します。LLMが利用できない場合は、"
"検索結果をそのまま表示します。" % _N_KB,
},
"badge_inventory": {
"en": f"🧪 <b>Source:</b> randomly generated by the developer's code (<code>src/data_generation.py</code>, fixed seed) — "
f"{len(inventory_df)} synthetic SKU records and {len(orders_df)} synthetic orders. Not exported from any real company's WMS.<br>"
"<b>What it describes:</b> a stand-in inventory table (SKU, category, zone, on-hand units, reorder point, unit cost) and "
"orders table (order ID, status, line count, priority, zone) — realistic in shape, but fictional records.<br>"
"<b>Function:</b> classifies whether a query is about inventory or order status, extracts SKU / order-ID / zone "
"identifiers with regex, and filters the matching table — a lightweight stand-in for a natural-language WMS query tool.",
"ja": f"🧪 <b>データの出典:</b>開発者のコード(<code>src/data_generation.py</code>、乱数シード固定)で生成した合成データ"
f"(SKU {len(inventory_df)}件、注文 {len(orders_df)}件)。実在企業のWMSから出力したものではありません。<br>"
"<b>内容:</b>在庫テーブル(SKU・カテゴリ・ゾーン・在庫数・発注点・単価)と注文テーブル(注文ID・ステータス・明細数・優先度・ゾーン)。"
"形式は現実的ですが、内容は架空のレコードです。<br>"
"<b>機能:</b>問い合わせが在庫か注文状況かを分類し、SKU・注文ID・ゾーンを正規表現で抽出して該当テーブルを絞り込みます。"
"自然言語によるWMS照会ツールの簡易版です。",
},
"badge_predictive": {
"en": "🧪 <b>Source:</b> randomly generated by the developer's code (<code>src/data_generation.py</code>, fixed seed) — 1,000 synthetic "
"sensor readings (900 normal + 100 simulated-fault). Not logged from real equipment.<br>"
"<b>What it describes:</b> conveyor/crane motor readings — temperature, vibration, current draw, belt speed — with 'anomaly' "
"readings modelled on failure signatures (elevated temp/vibration/current with reduced belt speed, as with bearing wear or overload).<br>"
"<b>Function:</b> an Isolation Forest is fit unsupervised (it never sees an anomaly label while fitting); its decision threshold "
"is then calibrated on a small labelled calibration set (never on test data). Check one reading with the sliders, or upload a CSV "
"to score a whole shift's logs. Anomalies can trigger a Slack alert (real if a <code>SLACK_WEBHOOK_URL</code> secret is set, "
"otherwise a clearly-labelled preview).",
"ja": "🧪 <b>データの出典:</b>開発者のコード(<code>src/data_generation.py</code>、乱数シード固定)で生成した合成センサーデータ1,000件"
"(正常900+故障模擬100)。実機のログではありません。<br>"
"<b>内容:</b>コンベヤ/クレーンのモーター計測値(温度・振動・電流・ベルト速度)。異常データは、ベアリング摩耗や過負荷に見られる"
"故障パターン(温度・振動・電流の上昇とベルト速度の低下)を模して作成しています。<br>"
"<b>機能:</b>Isolation Forestを教師なしで学習し(学習時に異常ラベルは使用しません)、判定しきい値は少量のラベル付き較正データで調整します"
"(テストデータは使用しません)。スライダーで1件を判定するほか、CSVをアップロードしてシフト全体のログを一括判定できます。"
"異常時はSlack通知が可能です(<code>SLACK_WEBHOOK_URL</code>設定時は実送信、未設定時はプレビュー表示)。",
},
"badge_vision": {
"en": "🧪 <b>Source:</b> <b>Real photos:</b> anything you upload (processed in memory, not stored), plus three openly-licensed "
"example photos fetched at runtime from Wikimedia Commons with author and license shown — no third-party images are bundled "
"in this repo. <b>Synthetic baseline:</b> rack images generated by <code>src/vision_model.py</code>, where ground truth is known.<br>"
"<b>What it describes:</b> real warehouse scenes (racking, pallets, forklifts, people) for the YOLO detectors; a simplified 3×6 storage rack "
"with box shapes for the baseline.<br>"
"<b>Function:</b> <i>Standard</i> mode runs COCO-pretrained YOLOv8, which finds people and vehicles but has <b>no pallet, forklift or box "
"class</b>. <i>Find by name</i> mode runs open-vocabulary YOLOE: type what to look for and it searches zero-shot. "
"<b>Accuracy on real warehouse photos has not been measured</b> — no labelled warehouse images were available — so treat counts as a "
"starting point. The classical-CV baseline on synthetic racks <i>is</i> measured (see Model Evaluation).",
"ja": "🧪 <b>データの出典:</b><b>実写真:</b>アップロードした画像(メモリ上で処理し、保存しません)と、Wikimedia Commonsから実行時に取得する"
"オープンライセンスの例示写真3枚(作者・ライセンスを表示)。第三者の画像はリポジトリに同梱していません。"
"<b>合成ベースライン:</b><code>src/vision_model.py</code>で生成した、正解が既知のラック画像。<br>"
"<b>内容:</b>YOLO検出器向けの実際の倉庫シーン(ラック、パレット、フォークリフト、作業者)と、ベースライン用の簡易な3×6ラック画像。<br>"
"<b>機能:</b><i>標準</i>モードはCOCO学習済みYOLOv8を使用し、人と車両は検出できますが、<b>パレット・フォークリフト・箱のクラスはありません</b>。"
"<i>名前で検索</i>モードはオープン語彙のYOLOEで、探したい物を文字で入力してゼロショット検出します。"
"<b>実際の倉庫写真での精度は未測定</b>です(ラベル付き倉庫画像が入手できなかったため)。件数は目安としてご利用ください。"
"合成ラックでの古典的画像処理ベースラインは測定済みです(Model Evaluationタブ参照)。",
},
"badge_eval": {
"en": "🧪 All charts below are computed on <b>held-out test data</b> by <code>build_artifacts.py</code> — fully reproducible, not cherry-picked.",
"ja": "🧪 以下の図表はすべて、<code>build_artifacts.py</code>が<b>ホールドアウトしたテストデータ</b>で算出したものです。再現可能で、結果の取捨選択は行っていません。",
},
}
def about_ja() -> str:
return f"""
# 🏭 スマート倉庫AIアシスタント
**物流・倉庫自動化の運用を題材に、実務的なAI活用を示すポートフォリオプロジェクトです。**
AS/RS、コンベヤ・仕分け、AGV/AMR、WMS/WCSといった倉庫自動化の現場では、設備テレメトリ、取引データ、安全記録、現場からの問い合わせなど
大量の運用データが生まれます。本プロジェクトは、その上にAIレイヤーを載せる最小構成の実装例です。依頼を正しく振り分け、推測ではなく
根拠のある情報で回答し、設備の異常を早期に検出し、さらに**再現可能で誠実な評価**を行うことを重視しています。
## 機能一覧
| 機能 | タブ | 主な技術 |
|---|---|---|
| ナレッジに基づく**LLM対話アシスタント**(RAG/マルチターン会話対応) | AI Assistant | TF-IDF検索 + ホスト型LLM |
| **意図分類**による問い合わせの振り分け(8カテゴリ) | AI Assistant / Inventory | TF-IDF + ロジスティック回帰 |
| 在庫・注文の**自然文照会** | Inventory & Order Query | 正規表現によるスロット抽出 + 意図分類 |
| センサーデータの**予知保全**(1件判定/CSV一括判定/Slack通知) | Predictive Maintenance | Isolation Forest(教師なし)+ 較正済みしきい値 |
| **実写真の物体検出**(人・車両/名前指定でパレット等)+合成ラックのベースライン | Visual Inventory Scan | YOLOv8 / YOLOE + 古典的画像処理 |
| 全コンポーネントの**再現可能な評価** | Model Evaluation | scikit-learn指標、ホールドアウト検証 |
## データセットの概要
**本プロジェクトのデータはすべて、自作コードによる合成データです。**実在企業のシステムからの取得・流用は一切ありません。
プライバシーやライセンスの懸念なく、誰でも再現・共有できます。
| データ | 内容 | 件数 | 生成方法 |
|---|---|---|---|
| ナレッジベース | 倉庫自動化の一般的概念を解説した手書き記事 | {_N_KB}記事 | 本プロジェクトのために作成 |
| 意図分類クエリ | 8カテゴリの問い合わせ例 | 約480件 | テンプレートにSKU・ゾーン・設備IDなどをランダムに埋め込み |
| 在庫テーブル | SKU・カテゴリ・ゾーン・在庫数・発注点・単価 | 60 SKU | 現実的な範囲でランダム生成(シード固定) |
| 注文テーブル | ステータス・明細数・優先度・ゾーン | 80件 | 現実的な範囲でランダム生成(シード固定) |
| センサーデータ | 温度・振動・電流・ベルト速度 | 1,000件(正常900/異常100) | 正常は運転範囲から、異常は故障パターンを模して生成 |
| ラック画像 | 箱の配置が既知の合成画像 | 随時生成 | PILで描画。ノイズ強度を調整可能 |
| 検索評価セット | 質問と正解記事のペア | {_N_KB}組 | 検索精度確認のため手作業で作成 |
## 各モデルの役割
| モデル | 目的 | アルゴリズム | 入力 → 出力 |
|---|---|---|---|
| 意図分類器 | 問い合わせの種類を判定 | TF-IDF + ロジスティック回帰 | 文章 → 意図ラベル + 確信度 |
| 検索(RAG) | 回答の根拠となる記事を取得 | TF-IDF + コサイン類似度 | 文章 → 関連記事の上位k件 |
| LLMアシスタント | 根拠に基づく自然な回答を生成 | ホスト型LLM(複数モデルのフォールバック) | 質問 + 根拠 + 会話履歴 → 回答 |
| 在庫・注文照会 | 質問をテーブル検索に変換 | 正規表現 + 意図分類 | 文章 → 絞り込み結果 |
| 異常検知 | 設備の異常な計測値を早期に検出 | Isolation Forest + 較正済みしきい値 | 4つのセンサー値 → 正常/異常 + スコア |
| 画像検出器 | 実写真から物体を検出 | YOLOv8(COCO)/YOLOE(名前指定)、ベースラインは古典的画像処理 | 画像 → 検出枠 + クラス別件数 |
## 試し方
1. **AI Assistant** — 例の質問を選ぶか自由に入力します。続けて「Zone Dの場合は?」などと尋ねると、直前の文脈を踏まえて回答します。
2. **Inventory & Order Query** — 「SKU-1042 は Zone B に何個ある?」「注文 #10007 の状況は?」などを試します。
3. **Predictive Maintenance** — プリセットまたはスライダーで1件を判定します。「CSV一括判定」でサンプルCSVをダウンロードし、そのままアップロードして一括判定も試せます。
4. **Visual Inventory Scan** — 実写真をアップロードするか例示写真を読み込み、「名前で検索」で *pallet, forklift* などを探します(実写真での精度は未測定)。合成ラックのタブでは、ノイズを上げて検出が難しくなる様子を確認できます。
5. **Model Evaluation** — すべての図表はテストデータで算出しており、`build_artifacts.py`を再実行すれば同じ数値を再現できます。
## 評価の信頼性について
開発中に2件のデータリーク(意図分類での学習/テスト間の文面重複、異常検知でのしきい値への正解率の混入)を発見・修正しました。
修正後の指標は、意図分類 正解率 {intent_eval.get('accuracy', 0):.1%}、異常検知 F1 {anomaly_eval.get('f1', 0):.1%} です。
合成ラックの古典的画像処理はノイズなしでは100%となるため、ノイズを加えた条件(標準ノイズ0.4で完全一致率 {vision_eval.get('exact_match_rate', 0):.0%})を報告しています。実写真に対するYOLOの精度は未測定です。
## 制約と今後の課題
- データはすべて**合成**です。実運用ではWMS/WCSのAPIや実際のセンサーログに接続します。
- 実写真向けのYOLOはゼロショット/汎用モデルであり、倉庫専用データでの微調整・評価は未実施です。精度を保証するには、ラベル付き倉庫画像(約100枚〜)でmAPを測定する必要があります。
- Slack通知は`SLACK_WEBHOOK_URL`未設定の場合はプレビューのみで、実際には送信されません。
"""
def data_badge(html: str) -> str:
return f'<div class="data-badge">{html}</div>'
with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue"), css=CUSTOM_CSS, title="Smart Warehouse AI Assistant") as demo:
gr.Markdown("<h1 id='title-banner'>🏭 Smart Warehouse AI Assistant</h1>")
subtitle_md = gr.Markdown(I18N["subtitle"]["en"])
lang_radio = gr.Radio(["English", "日本語"], value="English", label="🌐 Language / 言語", interactive=True)
if not LLM_CONFIGURED:
gr.Markdown(
"> ⚠️ **No LLM configured.** The AI Assistant tab runs in **retrieval-only fallback mode** "
"(still functional, just not LLM-generated prose). To enable full answers, add a Space secret "
"`GROQ_API_KEY` (free key at console.groq.com) — or `LLM_API_KEY` + `LLM_BASE_URL` + `LLM_MODEL` "
"for any OpenAI-compatible API, or an `HF_TOKEN` with Hugging Face inference credits."
)
with gr.Tab("💬 AI Assistant"):
with gr.Accordion("🧪 About this data & function / データ・機能について", open=False):
badge_assistant = gr.HTML(data_badge(I18N["badge_assistant"]["en"]))
gr.ChatInterface(
fn=chat_fn,
type="messages",
examples=ASSISTANT_EXAMPLES,
chatbot=gr.Chatbot(height=430, label="Warehouse Assistant", type="messages"),
textbox=gr.Textbox(placeholder="e.g. The conveyor belt in Zone C is making noise"),
)
with gr.Accordion("🔧 LLM connection diagnostics", open=False):
gr.Markdown(
"If the assistant keeps answering in retrieval-only fallback mode, "
"click below to test the LLM connection directly and see the exact "
"error from each candidate model."
)
test_llm_btn = gr.Button("Test LLM connection")
test_llm_output = gr.Markdown()
test_llm_btn.click(test_llm_fn, inputs=None, outputs=test_llm_output)
with gr.Tab("📦 Inventory & Order Query"):
with gr.Accordion("🧪 About this data & function / データ・機能について", open=False):
badge_inventory = gr.HTML(data_badge(I18N["badge_inventory"]["en"]))
with gr.Row():
inv_input = gr.Textbox(label="Query", placeholder="How many units of SKU-1042 are in Zone B?", scale=4)
inv_btn = gr.Button("Search", variant="primary", scale=1)
inv_note = gr.Markdown()
inv_result = gr.Dataframe(label="Results", wrap=True)
inv_btn.click(inventory_query_fn, inputs=inv_input, outputs=[inv_note, inv_result])
inv_input.submit(inventory_query_fn, inputs=inv_input, outputs=[inv_note, inv_result])
gr.Examples(
examples=[
"How many units of SKU-1042 are in Zone B?",
"What's the status of order #10007?",
"Show me low stock items",
"Any delayed orders?",
"Is SKU-1015 in stock at Zone A?",
"Do we have enough SKU-1030 to fulfill 200 units?",
"Track order #10021 for me",
"Show the fulfillment status of #10045",
],
inputs=inv_input,
)
with gr.Accordion("Browse full tables", open=False):
gr.Markdown("**Inventory** (synthetic)")
gr.Dataframe(inventory_df, wrap=True)
gr.Markdown("**Orders** (synthetic)")
gr.Dataframe(orders_df, wrap=True)
with gr.Tab("⚠️ Predictive Maintenance"):
with gr.Accordion("🧪 About this data & function / データ・機能について", open=False):
badge_predictive = gr.HTML(data_badge(I18N["badge_predictive"]["en"]))
preset_dropdown = gr.Dropdown(
choices=list(ANOMALY_PRESETS.keys()), label="Load a preset reading", value="Normal reading"
)
with gr.Row():
motor_temp_in = gr.Slider(30, 100, value=55, step=0.5, label="Motor temperature (°C)")
vibration_in = gr.Slider(0, 10, value=2.2, step=0.1, label="Vibration (mm/s)")
with gr.Row():
current_in = gr.Slider(5, 30, value=12, step=0.5, label="Motor current (A)")
belt_speed_in = gr.Slider(0.1, 2.5, value=1.5, step=0.05, label="Belt speed (m/s)")
with gr.Row():
zone_label_in = gr.Textbox(label="Equipment / zone label (used in alerts)", placeholder="e.g. Conveyor-14, Zone C", scale=3)
send_alert_in = gr.Checkbox(label="Send Slack alert if anomalous", value=False, scale=1)
check_btn = gr.Button("Check for anomaly", variant="primary")
anomaly_output = gr.Markdown()
preset_dropdown.change(
load_preset, inputs=preset_dropdown,
outputs=[motor_temp_in, vibration_in, current_in, belt_speed_in],
)
check_btn.click(
anomaly_fn,
inputs=[motor_temp_in, vibration_in, current_in, belt_speed_in, zone_label_in, send_alert_in],
outputs=anomaly_output,
)
with gr.Accordion("📋 Batch mode: upload a CSV of sensor readings", open=False):
gr.Markdown(
f"Upload a CSV with columns `{'`, `'.join(SENSOR_FEATURES)}` (optionally an `id` column) to run "
"detection over many readings at once — e.g. an export of the last shift's sensor logs. "
"Don't have one? Download the sample below and upload it back."
)
gr.File(value=SAMPLE_BATCH_CSV, label="Sample CSV (40 readings, ~15% faulty)", interactive=False)
batch_csv_in = gr.File(label="Your sensor readings CSV", file_types=[".csv"])
batch_btn = gr.Button("Run batch detection")
batch_summary = gr.Markdown()
batch_results = gr.Dataframe(label="Results", wrap=True)
batch_download = gr.File(label="Download results CSV")
batch_btn.click(anomaly_batch_fn, inputs=batch_csv_in,
outputs=[batch_summary, batch_results, batch_download])
with gr.Tab("📷 Visual Inventory Scan"):
with gr.Accordion("🧪 About this data & function / データ・機能について", open=False):
badge_vision = gr.HTML(data_badge(I18N["badge_vision"]["en"]))
with gr.Tabs():
with gr.Tab("📸 Real photos (YOLO)"):
gr.Markdown(
"Upload a real warehouse photo — or load an openly-licensed example — and run a YOLO "
"detector. **Standard** mode uses COCO-pretrained YOLOv8 (finds people and vehicles; "
"COCO has no pallet/forklift/box class). **Find by name** mode uses open-vocabulary "
"YOLOE: type what to look for (*pallet, cardboard box, forklift…*) and it searches "
"zero-shot. *The first run downloads model weights (≈1–2 minutes the first time in Find-by-name mode, then about a second).* "
"Photos are processed in memory and not stored — please don't upload anything sensitive."
)
with gr.Row():
with gr.Column(scale=1):
real_in = gr.Image(label="Warehouse photo", type="pil", sources=["upload", "clipboard"])
example_dd = gr.Dropdown(choices=list(EXAMPLE_PHOTOS), value=None,
label="…or an openly-licensed example (Wikimedia Commons)")
load_ex_btn = gr.Button("Load example")
mode_radio = gr.Radio(
choices=[("Find by name — YOLOE (open-vocabulary)", "open"),
("Standard objects — YOLOv8 (COCO)", "standard")],
value="open", label="Detector")
prompts_in = gr.Textbox(value=DEFAULT_PROMPTS, label="What to look for (comma-separated)", visible=True)
model_dd = gr.Dropdown(choices=list(MODEL_CHOICES), value=list(MODEL_CHOICES)[0],
label="YOLOv8 size", visible=False)
conf_in = gr.Slider(0.05, 0.9, value=0.25, step=0.05, label="Confidence threshold")
iou_in = gr.Slider(0.3, 0.9, value=0.7, step=0.05,
label="Overlap suppression (IoU) — lower merges duplicate boxes, but may merge touching pallets")
detect_btn = gr.Button("🔍 Detect", variant="primary")
with gr.Column(scale=1):
real_out = gr.Image(label="Detections", type="pil")
real_summary = gr.Markdown()
example_credit = gr.Markdown()
mode_radio.change(
lambda m: (gr.update(visible=(m == "open")), gr.update(visible=(m == "standard"))),
inputs=mode_radio, outputs=[prompts_in, model_dd])
load_ex_btn.click(load_example_fn, inputs=example_dd, outputs=[real_in, example_credit])
detect_btn.click(real_detect_fn, inputs=[real_in, mode_radio, model_dd, prompts_in, conf_in, iou_in],
outputs=[real_out, real_summary])
with gr.Tab("🧪 Synthetic rack (classical-CV baseline)"):
gr.Markdown(
"The original baseline: a *non-deep-learning* OpenCV pipeline on synthetic rack images "
"where ground truth is known, so accuracy can be measured exactly. Raise the noise slider "
"to watch it degrade the way a dim or grainy real photo would."
)
with gr.Row():
fill_density_in = gr.Slider(0.1, 0.95, value=0.6, step=0.05, label="Rack fill density (scene generation)")
noise_in = gr.Slider(0.0, 1.0, value=0.3, step=0.05, label="Simulated photographic noise")
vision_seed_state = gr.State(value=1)
new_scene_btn = gr.Button("🎲 Generate new rack scene", variant="primary")
with gr.Row():
vision_raw_img = gr.Image(label="Rack photo (input)", type="pil")
vision_annotated_img = gr.Image(label="Detected boxes (output)", type="pil")
vision_summary = gr.Markdown()
_vision_io = dict(inputs=[fill_density_in, noise_in, vision_seed_state],
outputs=[vision_raw_img, vision_annotated_img, vision_summary, vision_seed_state])
new_scene_btn.click(vision_scan_fn, **_vision_io)
demo.load(vision_scan_fn, **_vision_io)
with gr.Tab("📊 Model Evaluation"):
with gr.Accordion("🧪 About this data & function / データ・機能について", open=False):
badge_eval = gr.HTML(data_badge(I18N["badge_eval"]["en"]))
gr.Markdown("### 1. Intent Classifier — accuracy: **{:.1%}** · macro F1: **{:.1%}**".format(
intent_eval.get("accuracy", 0), intent_eval.get("macro_f1", 0)
))
with gr.Row():
gr.Image(os.path.join(ASSETS_DIR, "intent_dataset_composition.png"), show_label=False, container=False)
gr.Image(os.path.join(ASSETS_DIR, "intent_per_class_bar.png"), show_label=False, container=False)
gr.Image(os.path.join(ASSETS_DIR, "intent_confusion_matrix.png"), show_label=False, container=False)
gr.Markdown("### 2. Predictive Maintenance — F1: **{:.1%}** · ROC-AUC: **{:.2f}**".format(
anomaly_eval.get("f1", 0), anomaly_eval.get("roc_auc", 0)
))
gr.Image(os.path.join(ASSETS_DIR, "sensor_distributions.png"), show_label=False, container=False)
with gr.Row():
gr.Image(os.path.join(ASSETS_DIR, "anomaly_metrics_bar.png"), show_label=False, container=False)
gr.Image(os.path.join(ASSETS_DIR, "anomaly_confusion_matrix.png"), show_label=False, container=False)
gr.Image(os.path.join(ASSETS_DIR, "anomaly_roc_curve.png"), show_label=False, container=False)
gr.Markdown("### 3. RAG Retrieval — Hit-rate@1: **{:.0%}** · Hit-rate@2: **{:.0%}**".format(
retrieval_eval.get("hit_rate_at_1", 0), retrieval_eval.get("hit_rate_at_2", 0)
))
gr.Image(os.path.join(ASSETS_DIR, "retrieval_hitrate_bar.png"), show_label=False, container=False)
gr.Markdown("### 4. Latency Benchmark (CPU, local components)")
gr.Image(os.path.join(ASSETS_DIR, "latency_bar.png"), show_label=False, container=False)
gr.Markdown("### 5. Visual Inventory Scan — synthetic-rack classical baseline (measured): exact box-count match **{:.0%}** (noise={}) · count MAE **{:.2f}** boxes".format(
vision_eval.get("exact_match_rate", 0), vision_eval.get("standard_noise_level", "?"), vision_eval.get("count_mae", 0)
))
with gr.Row():
gr.Image(os.path.join(ASSETS_DIR, "vision_metrics_bar.png"), show_label=False, container=False)
gr.Image(os.path.join(ASSETS_DIR, "vision_noise_robustness.png"), show_label=False, container=False)
with gr.Row():
gr.Image(os.path.join(ASSETS_DIR, "vision_sample_raw.png"), label="Sample input", container=True)
gr.Image(os.path.join(ASSETS_DIR, "vision_sample_annotated.png"), label="Sample detection", container=True)
gr.Markdown(
"### 5b. Real-photo detectors (YOLOv8 / YOLOE) — **not benchmarked**\n"
"No labelled real warehouse photos were available, so there is deliberately **no accuracy number** here. "
"COCO benchmark scores published for YOLOv8 describe everyday objects, not pallets or racking, so they "
"would be misleading to quote. To measure it properly: label ~100 warehouse photos (pallets, forklifts, "
"people), then report per-class precision / recall / mAP50 — and compare zero-shot YOLOE against a YOLOv8 "
"fine-tuned on that data."
)
with gr.Accordion("📋 Full metrics tables & methodology notes", open=False):
gr.Markdown(eval_intent_section_md())
gr.Markdown(eval_anomaly_section_md())
gr.Markdown(eval_retrieval_section_md())
gr.Markdown(eval_latency_section_md())
gr.Markdown(EVAL_METHODOLOGY_MD)
with gr.Tab("ℹ️ About"):
about_md = gr.Markdown(ABOUT_MD)
def set_language(lang):
k = "ja" if lang == "日本語" else "en"
return (
I18N["subtitle"][k],
data_badge(I18N["badge_assistant"][k]),
data_badge(I18N["badge_inventory"][k]),
data_badge(I18N["badge_predictive"][k]),
data_badge(I18N["badge_vision"][k]),
data_badge(I18N["badge_eval"][k]),
about_ja() if k == "ja" else ABOUT_MD,
)
lang_radio.change(
set_language, inputs=lang_radio,
outputs=[subtitle_md, badge_assistant, badge_inventory, badge_predictive,
badge_vision, badge_eval, about_md],
)
if __name__ == "__main__":
# ssr_mode=False: Gradio's experimental server-side rendering mode was
# causing repeated "SvelteKitError: POST method not allowed" / 405 log
# spam on Spaces (some requests hit the SSR page route instead of the
# API route). Disabling it avoids that; this app has no functional need
# for SSR. Wrapped defensively in case the kwarg name differs across
# Gradio versions -- falls back to a plain launch() rather than crashing
# the Space if so.
launch_kwargs = dict(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)))
try:
demo.launch(**launch_kwargs, ssr_mode=False)
except TypeError:
demo.launch(**launch_kwargs)
"""
Smart Warehouse AI Assistant
=============================
A warehouse/intralogistics AI copilot demo, built for a Hugging Face
Space. Combines:
1. An LLM-powered assistant (RAG: TF-IDF retrieval + hosted LLM via the
an OpenAI-compatible API) for natural-language warehouse operations Q&A.
2. A TF-IDF + Logistic Regression intent classifier that routes queries
into 8 operational categories.
3. A lightweight NL -> structured query layer over synthetic inventory /
order tables.
4. An Isolation Forest anomaly detector for conveyor/crane sensor
streams (predictive maintenance).
5. A Model Evaluation tab reporting real accuracy/F1/ROC-AUC metrics
computed by build_artifacts.py.
Author: (your name here) -- built as a portfolio / job-application project.
"""
import json
import os
import gradio as gr
import pandas as pd
from src.anomaly_model import FEATURES as SENSOR_FEATURES
from src.anomaly_model import load_artifacts as load_anomaly_artifacts
from src.anomaly_model import score_reading
from src.alerting import send_anomaly_alert
from src.vision_model import generate_rack_scene, detect_boxes
from src.real_vision import (
DEFAULT_PROMPTS, EXAMPLE_PHOTOS, MODEL_CHOICES, detect_real, fetch_example,
summarize as summarize_real, vision_stack_available,
)
from src.data_generation import generate_inventory_db, generate_orders_db
from src.intent_model import INTENT_DESCRIPTIONS, load_pipeline, predict as intent_predict
from src.inventory_db import query_inventory, query_orders
from src.llm_client import answer_query, test_connection, llm_configured
from src.retriever import KBRetriever
ROOT = os.path.dirname(os.path.abspath(__file__))
MODELS_DIR = os.path.join(ROOT, "models")
DATA_DIR = os.path.join(ROOT, "data")
ASSETS_DIR = os.path.join(ROOT, "assets")
# --------------------------------------------------------------------------
# Load pre-trained artifacts (fast: no training happens at Space startup
# under normal conditions). If unpickling fails -- e.g. the models were
# built with a different scikit-learn version than the one installed in
# this container -- we transparently rebuild everything from scratch. This
# is fast (a few seconds, see build_artifacts.py) and makes the app immune
# to sklearn version-pinning mismatches between build time and deploy time.
# --------------------------------------------------------------------------
def _load_or_rebuild_artifacts():
intent_path = os.path.join(MODELS_DIR, "intent_pipeline.joblib")
anomaly_model_path = os.path.join(MODELS_DIR, "anomaly_iforest.joblib")
anomaly_scaler_path = os.path.join(MODELS_DIR, "anomaly_scaler.joblib")
anomaly_threshold_path = os.path.join(MODELS_DIR, "anomaly_threshold.joblib")
def _try_load():
pipeline = load_pipeline(intent_path)
model, scaler, threshold = load_anomaly_artifacts(anomaly_model_path, anomaly_scaler_path, threshold_path=anomaly_threshold_path)
# Smoke-test the loaded pipeline against the exact code path the app
# uses at request time. On a scikit-learn version mismatch, sklearn
# sometimes unpickles "successfully" but throws later on first real
# use (e.g. LogisticRegression missing an internal attribute) --
# catching that here, at import time, is what makes this self-healing.
intent_predict(pipeline, "healthcheck")
score_reading(model, scaler, {"motor_temp_c": 55, "vibration_mm_s": 2.2, "current_amps": 12, "belt_speed_mps": 1.5}, threshold=threshold)
return pipeline, model, scaler, threshold
try:
return _try_load()
except Exception as e: # noqa: BLE001 -- any load/version issue triggers a rebuild
print(f"[startup] Could not load pre-built model artifacts ({type(e).__name__}: {e}). "
f"Rebuilding from scratch with the installed scikit-learn version...")
import build_artifacts
build_artifacts.build_intent_classifier()
build_artifacts.build_anomaly_detector()
build_artifacts.build_retrieval_eval()
build_artifacts.build_inventory_and_orders()
print("[startup] Rebuild complete.")
return _try_load()
intent_pipeline, anomaly_model, anomaly_scaler, anomaly_threshold = _load_or_rebuild_artifacts()
retriever = KBRetriever()
# Prefer the pre-generated CSVs (so demo state matches the eval run); fall back to
# regenerating in-memory if they're missing for some reason.
try:
inventory_df = pd.read_csv(os.path.join(DATA_DIR, "inventory.csv"))
orders_df = pd.read_csv(os.path.join(DATA_DIR, "orders.csv"))
except FileNotFoundError:
inventory_df = generate_inventory_db()
orders_df = generate_orders_db()