""" Smart Warehouse AI Assistant ============================= A warehouse/intralogistics AI copilot demo, built for a Hugging Face Space. Combines: 1. An LLM-powered assistant (RAG: TF-IDF retrieval + hosted LLM via the an OpenAI-compatible API) for natural-language warehouse operations Q&A. 2. A TF-IDF + Logistic Regression intent classifier that routes queries into 8 operational categories. 3. A lightweight NL -> structured query layer over synthetic inventory / order tables. 4. An Isolation Forest anomaly detector for conveyor/crane sensor streams (predictive maintenance). 5. A Model Evaluation tab reporting real accuracy/F1/ROC-AUC metrics computed by build_artifacts.py. Author: (your name here) -- built as a portfolio / job-application project. """ import json import os import gradio as gr import pandas as pd from src.anomaly_model import FEATURES as SENSOR_FEATURES from src.anomaly_model import load_artifacts as load_anomaly_artifacts from src.anomaly_model import score_reading from src.alerting import send_anomaly_alert from src.vision_model import generate_rack_scene, detect_boxes from src.real_vision import ( DEFAULT_PROMPTS, EXAMPLE_PHOTOS, MODEL_CHOICES, detect_real, fetch_example, summarize as summarize_real, vision_stack_available, ) from src.data_generation import generate_inventory_db, generate_orders_db from src.intent_model import INTENT_DESCRIPTIONS, load_pipeline, predict as intent_predict from src.inventory_db import query_inventory, query_orders from src.llm_client import answer_query, test_connection, llm_configured from src.retriever import KBRetriever ROOT = os.path.dirname(os.path.abspath(__file__)) MODELS_DIR = os.path.join(ROOT, "models") DATA_DIR = os.path.join(ROOT, "data") ASSETS_DIR = os.path.join(ROOT, "assets") # -------------------------------------------------------------------------- # Load pre-trained artifacts (fast: no training happens at Space startup # under normal conditions). If unpickling fails -- e.g. the models were # built with a different scikit-learn version than the one installed in # this container -- we transparently rebuild everything from scratch. This # is fast (a few seconds, see build_artifacts.py) and makes the app immune # to sklearn version-pinning mismatches between build time and deploy time. # -------------------------------------------------------------------------- def _load_or_rebuild_artifacts(): intent_path = os.path.join(MODELS_DIR, "intent_pipeline.joblib") anomaly_model_path = os.path.join(MODELS_DIR, "anomaly_iforest.joblib") anomaly_scaler_path = os.path.join(MODELS_DIR, "anomaly_scaler.joblib") anomaly_threshold_path = os.path.join(MODELS_DIR, "anomaly_threshold.joblib") def _try_load(): pipeline = load_pipeline(intent_path) model, scaler, threshold = load_anomaly_artifacts(anomaly_model_path, anomaly_scaler_path, threshold_path=anomaly_threshold_path) # Smoke-test the loaded pipeline against the exact code path the app # uses at request time. On a scikit-learn version mismatch, sklearn # sometimes unpickles "successfully" but throws later on first real # use (e.g. LogisticRegression missing an internal attribute) -- # catching that here, at import time, is what makes this self-healing. intent_predict(pipeline, "healthcheck") score_reading(model, scaler, {"motor_temp_c": 55, "vibration_mm_s": 2.2, "current_amps": 12, "belt_speed_mps": 1.5}, threshold=threshold) return pipeline, model, scaler, threshold try: return _try_load() except Exception as e: # noqa: BLE001 -- any load/version issue triggers a rebuild print(f"[startup] Could not load pre-built model artifacts ({type(e).__name__}: {e}). " f"Rebuilding from scratch with the installed scikit-learn version...") import build_artifacts build_artifacts.build_intent_classifier() build_artifacts.build_anomaly_detector() build_artifacts.build_retrieval_eval() build_artifacts.build_inventory_and_orders() print("[startup] Rebuild complete.") return _try_load() intent_pipeline, anomaly_model, anomaly_scaler, anomaly_threshold = _load_or_rebuild_artifacts() retriever = KBRetriever() # Prefer the pre-generated CSVs (so demo state matches the eval run); fall back to # regenerating in-memory if they're missing for some reason. try: inventory_df = pd.read_csv(os.path.join(DATA_DIR, "inventory.csv")) orders_df = pd.read_csv(os.path.join(DATA_DIR, "orders.csv")) except FileNotFoundError: inventory_df = generate_inventory_db() orders_df = generate_orders_db() def _load_json(name): path = os.path.join(DATA_DIR, name) if os.path.exists(path): with open(path) as f: return json.load(f) return {} intent_eval = _load_json("intent_eval.json") anomaly_eval = _load_json("anomaly_eval.json") retrieval_eval = _load_json("retrieval_eval.json") latency_eval = _load_json("latency_eval.json") vision_eval = _load_json("vision_eval.json") LLM_CONFIGURED = llm_configured() # -------------------------------------------------------------------------- # ZeroGPU compatibility shim # -------------------------------------------------------------------------- # This app is CPU-only by design (scikit-learn locally, LLM calls go to the # remote LLM API). Some Spaces accounts, however, only offer the # free "ZeroGPU" hardware tier, which requires at least one function # decorated with `@spaces.GPU` to be present or the platform's startup # check fails with "No @spaces.GPU function detected". This is a harmless, # unused health-check function that satisfies that requirement without # changing any real behaviour -- it is never called on the request path. try: import spaces @spaces.GPU(duration=5) def _zerogpu_healthcheck(): return True except ImportError: # Running locally / on CPU-basic hardware where `spaces` isn't installed. def _zerogpu_healthcheck(): return True # ========================================================================== # TAB 1 -- AI Assistant (LLM + RAG + intent routing) # ========================================================================== def chat_fn(message, history): intent = intent_predict(intent_pipeline, message) # `history` (Gradio messages format) is passed through so the LLM can handle # follow-ups like "what about Zone D instead?" (bounded/cleaned in llm_client). response = answer_query(message, retriever, history=history) intent_label = INTENT_DESCRIPTIONS.get(intent.intent, intent.intent) meta_lines = [f"**Detected intent:** {intent_label} ({intent.confidence:.0%} confidence)"] if response.sources: src_str = ", ".join(f"{s.title} ({s.score:.2f})" for s in response.sources) meta_lines.append(f"**Retrieved context:** {src_str}") meta_lines.append( f"**Generation:** {'LLM (' + response.model_id + ')' if response.used_llm else 'retrieval-only fallback'}" f" · {response.latency_s * 1000:.0f} ms" ) if not response.used_llm and response.debug_errors: err_lines = "\n".join(f" - `{e}`" for e in response.debug_errors) meta_lines.append(f"**Why the LLM wasn't used:**\n{err_lines}") full_reply = response.answer + "\n\n---\n" + "\n".join(meta_lines) return full_reply def test_llm_fn(): response = test_connection(retriever) if response.used_llm: return ( f"✅ **LLM connection working.** Model: `{response.model_id}` · " f"{response.latency_s * 1000:.0f} ms\n\nSample answer: {response.answer}" ) err_lines = "\n".join(f"- {e}" for e in response.debug_errors) or "(no error detail captured)" return ( "❌ **LLM connection failed** — running in retrieval-only fallback mode.\n\n" f"**What each configured backend reported:**\n{err_lines}\n\n" "**Most reliable fix:** add a `GROQ_API_KEY` Space secret (free key at console.groq.com), then " "restart the Space. Hugging Face's router needs inference credits, and free accounts get almost none " "(that's the `402 Payment Required` error). Any OpenAI-compatible API also works via " "`LLM_API_KEY` + `LLM_BASE_URL` + `LLM_MODEL`." ) ASSISTANT_EXAMPLES = [ "The conveyor belt in Zone C is making noise", "How many units of SKU-1042 are in Zone B?", "What's the status of order #10007?", "What's the difference between an AGV and an AMR?", "A forklift near-miss was reported in Zone A", "What's the fastest picking route for a high volume order?", "Is Crane-03 operational?", "Route AGV-12 to picking station 5", "What is cycle counting?", "Redirect AGV-07 around the blocked aisle in Zone B", "How does an AS/RS crane retrieve a pallet?", "What KPIs matter most in warehouse automation?", "Schedule maintenance for Sorter-02", "Should we batch pick these orders together?", "What is predictive maintenance?", "Log a safety incident involving AMR-21", ] # ========================================================================== # TAB 2 -- Inventory & Order Query # ========================================================================== def inventory_query_fn(text): intent = intent_predict(intent_pipeline, text) if intent.intent == "order_status": result = query_orders(orders_df, text) note = "Interpreted as an **order status** query." else: result = query_inventory(inventory_df, text) note = "Interpreted as an **inventory** query." if result.empty: result = pd.DataFrame({"message": ["No matching records found for this query."]}) return note, result # ========================================================================== # TAB 3 -- Predictive Maintenance / Anomaly Detection # ========================================================================== def anomaly_fn(motor_temp, vibration, current, belt_speed, zone_label="", send_alert=False): reading = { "motor_temp_c": motor_temp, "vibration_mm_s": vibration, "current_amps": current, "belt_speed_mps": belt_speed, } result = score_reading(anomaly_model, anomaly_scaler, reading, threshold=anomaly_threshold) verdict = "🔴 ANOMALY DETECTED" if result.is_anomaly else "🟢 Normal operating range" detail = ( f"### {verdict}\n\n" f"**Anomaly score:** {result.anomaly_score:.2f} / 1.00\n\n" f"| Sensor | Value | Typical normal range |\n|---|---|---|\n" f"| Motor temperature | {motor_temp:.1f} °C | 47–63 °C |\n" f"| Vibration | {vibration:.2f} mm/s | 1.2–3.2 mm/s |\n" f"| Motor current | {current:.1f} A | 9–15 A |\n" f"| Belt speed | {belt_speed:.2f} m/s | 1.2–1.8 m/s |\n" ) if result.is_anomaly: detail += ( "\n**Recommended action:** Flag for inspection. Elevated temperature + " "vibration + current with reduced belt speed typically indicates bearing " "wear, belt misalignment, or motor overload -- schedule maintenance before " "the next shift to avoid an unplanned stoppage." ) if send_alert: alert = send_anomaly_alert(zone_label or "Unspecified zone", reading, result.anomaly_score) if alert.sent: detail += "\n\n---\n✅ **Alert sent to Slack.**" else: why = alert.error or "no `SLACK_WEBHOOK_URL` secret configured" detail += (f"\n\n---\n📋 **Alert preview (NOT actually sent — {why}):**\n" f"```\n{alert.message_preview}\n```") return detail ANOMALY_PRESETS = { "Normal reading": (55.0, 2.2, 12.0, 1.5), "Early bearing wear": (68.0, 3.8, 15.5, 1.3), "Severe fault (imminent failure)": (85.0, 6.5, 21.0, 0.6), } def load_preset(name): return ANOMALY_PRESETS[name] def anomaly_batch_fn(csv_file): """CSV batch mode: run anomaly detection over many readings at once (e.g. an export of the last shift's sensor logs).""" if csv_file is None: return "Upload a CSV first.", None, None try: batch_df = pd.read_csv(csv_file.name if hasattr(csv_file, "name") else csv_file) except Exception as e: # noqa: BLE001 return f"Could not read CSV: {type(e).__name__}: {e}", None, None missing = [c for c in SENSOR_FEATURES if c not in batch_df.columns] if missing: return (f"CSV is missing column(s): {', '.join(missing)}. Expected: " f"{', '.join(SENSOR_FEATURES)} (optionally plus an `id` column).", None, None) rows = [] for _, row in batch_df.iterrows(): reading = {f: float(row[f]) for f in SENSOR_FEATURES} r = score_reading(anomaly_model, anomaly_scaler, reading, threshold=anomaly_threshold) rows.append({**({"id": row["id"]} if "id" in batch_df.columns else {}), **reading, "anomaly_score": round(r.anomaly_score, 3), "is_anomaly": r.is_anomaly}) out = pd.DataFrame(rows) n_bad = int(out["is_anomaly"].sum()) summary = f"**Processed {len(out)} readings** — {n_bad} flagged anomalous ({n_bad / max(len(out), 1):.1%})." path = os.path.join("/tmp", "anomaly_batch_results.csv") out.to_csv(path, index=False) return summary, out, path SAMPLE_BATCH_CSV = os.path.join(DATA_DIR, "sample_sensor_batch.csv") if not os.path.exists(SAMPLE_BATCH_CSV): # self-heal if the packaged sample is ever missing import build_artifacts build_artifacts.build_sample_batch_csv() # ========================================================================== # TAB 4 -- Visual Inventory Scan (classical computer vision) # ========================================================================== def vision_scan_fn(fill_density, noise_level, seed_state): seed = int(seed_state) if seed_state is not None else 1 scene = generate_rack_scene(fill_prob=fill_density, seed=seed, noise_level=noise_level) result = detect_boxes(scene.image) diff = abs(result.detected_count - scene.true_box_count) verdict = "✅ Exact match" if diff == 0 else f"⚠️ Off by {diff}" summary = ( f"### Detected **{result.detected_count} boxes** · estimated occupancy **{result.estimated_occupancy:.0%}**\n\n" f"| | CV pipeline | Ground truth |\n|---|---|---|\n" f"| Box count | {result.detected_count} | {scene.true_box_count} |\n" f"| Occupancy | {result.estimated_occupancy:.0%} | {scene.true_occupancy:.0%} |\n\n" f"**{verdict}** — ground truth exists only because this scene is synthetic; " f"a real deployment wouldn't have it." ) return scene.image, result.annotated_image, summary, seed + 1 def real_detect_fn(image, mode, model_choice, prompts_text, conf, iou=0.7): """Run a real-photo detector (YOLO) on an uploaded / example image.""" if image is None: return None, "⬆️ Upload a photo, or load an example, first." ok, why = vision_stack_available() if not ok: return None, (f"⚠️ The detector stack (ultralytics + PyTorch) isn't available on this server " f"(`{why}`). The rest of the app is unaffected.") try: res = detect_real(image, mode=mode, model_choice=model_choice, prompts_text=prompts_text, conf=conf, iou=iou) except ValueError as e: # bad user input (e.g. empty prompt box) -> plain guidance return None, f"✏️ {e}" except Exception as e: # noqa: BLE001 return None, (f"⚠️ Detection failed: `{type(e).__name__}: {e}`\n\n" f"The first run downloads model weights from GitHub, so a network problem there " f"is the most common cause — try again, or switch detector mode.") return res.annotated, summarize_real(res) def load_example_fn(name): img, msg = fetch_example(name) return img, msg # ========================================================================== # TAB 4 -- Model Evaluation # ========================================================================== def eval_intent_section_md(): return ( f"### 1. Intent Classifier " f"🎯 **{intent_eval.get('accuracy', 0):.0%} accuracy** · " f"**{intent_eval.get('macro_f1', 0):.0%} macro F1** · " f"held-out test set, {intent_eval.get('n_test', '?')} examples, " f"{intent_eval.get('n_classes', '?')} classes" ) def eval_anomaly_section_md(): return ( f"### 2. Predictive Maintenance (Anomaly Detector) " f"🎯 **{anomaly_eval.get('f1', 0):.0%} F1** · " f"**{anomaly_eval.get('roc_auc', 0):.2f} ROC-AUC** · " f"**{anomaly_eval.get('precision', 0):.0%} precision** · " f"**{anomaly_eval.get('recall', 0):.0%} recall**" ) def eval_retrieval_section_md(): return ( f"### 3. RAG Retrieval " f"🎯 **{retrieval_eval.get('hit_rate_at_1', 0):.0%} hit-rate@1** · " f"**{retrieval_eval.get('hit_rate_at_2', 0):.0%} hit-rate@2** · " f"{retrieval_eval.get('n_queries', '?')} labelled test queries" ) def eval_latency_section_md(): return ( f"### 4. Latency (CPU, avg of 50 runs) " f"🎯 **{latency_eval.get('intent_classifier_ms', '?')} ms** intent · " f"**{latency_eval.get('anomaly_detector_ms', '?')} ms** anomaly · " f"**{latency_eval.get('kb_retrieval_ms', '?')} ms** retrieval" ) EVAL_METHODOLOGY_MD = ( "*All metrics above are computed on held-out synthetic test data by " "`build_artifacts.py` (fully reproducible) — see the **About** tab for " "dataset details and how each model works. Two safeguards keep these " "numbers honest: (1) train/test splits are deduplicated on exact text " "with an automated zero-overlap check, so the intent classifier can't " "memorize verbatim test examples; (2) the anomaly detector is fit " "unsupervised on a training split, then its decision threshold is " "calibrated on a separate small *labelled calibration split* (analogous " "to a handful of confirmed historical incidents) — never on the test " "set used to report the metrics above.*" ) # ========================================================================== # TAB 5 -- About # ========================================================================== ABOUT_MD = f""" # 🏭 Smart Warehouse AI Assistant **A portfolio project demonstrating an applied-AI approach to intralogistics and warehouse automation operations.** Modern warehouse automation platforms — automated storage & retrieval systems (AS/RS), conveyor & sortation lines, AGVs/AMRs, and the WMS/WCS software that orchestrates them — generate huge volumes of operational data: equipment telemetry, transactions, safety logs, and ad-hoc questions from floor staff. This project is a compact, end-to-end example of how an AI layer can sit on top of that kind of system: routing requests correctly, answering from real operational context instead of guessing, catching equipment problems early, and doing all of it with **honest, reproducible evaluation** rather than a demo that just "looks like it works." ## What this demonstrates | Capability | Where | Techniques used | |---|---|---| | **LLM-powered natural-language assistant**, grounded with retrieval (RAG) so it answers from real domain knowledge rather than hallucinating | *AI Assistant* tab | TF-IDF retrieval + hosted LLM (HF Inference API) | | **Intent classification** to route free-text requests (maintenance, safety, navigation, inventory, etc.) the way a real ops system would triage tickets | *AI Assistant* / *Inventory* tabs | TF-IDF + Logistic Regression | | **Predictive maintenance** via unsupervised anomaly detection on conveyor/crane sensor streams — catching bearing wear or misalignment before an unplanned stoppage | *Predictive Maintenance* tab | Isolation Forest, unsupervised | | **NL-to-structured-query** over inventory/order data, a lightweight stand-in for a WMS query tool | *Inventory & Order Query* tab | Regex slot extraction + intent routing | | **Rigorous, reproducible evaluation** of every ML component (accuracy, F1, ROC-AUC, retrieval hit-rate, latency) rather than just a demo that "looks like it works" | *Model Evaluation* tab | scikit-learn metrics, held-out test splits | ## Dataset overview — what the data is and where it's from **Every dataset in this project is synthetically generated by the project's own code** (`src/data_generation.py` and `src/knowledge_base.py`), not scraped, exported, or sourced from any real company's systems. This was a deliberate choice: it keeps the project fully self-contained, reproducible, and shareable without any data-privacy or licensing concerns, while still being realistic enough to demonstrate the underlying ML techniques properly. | Dataset | What it is | Size | How it's generated | |---|---|---|---| | **Knowledge base** | Original, hand-written articles on generic warehouse-automation concepts (AS/RS, AGV/AMR, WMS, sortation, picking strategy, predictive maintenance, safety, inventory accuracy, KPIs, energy efficiency, cross-docking, returns, RFID, dock scheduling, slotting, security, labor management) | 17 articles | Written by the developer specifically for this project | | **Intent queries** | Example free-text operational questions/requests across 8 categories | ~480 examples | ~8 templates per category with randomised SKU codes, zone names, order IDs, and equipment IDs slotted in | | **Inventory table** | SKU records with category, zone, on-hand units, reorder point, unit cost | 60 SKUs | Randomised within realistic ranges (fixed seed) | | **Orders table** | Order records with status, line count, priority, zone | 80 orders | Randomised within realistic ranges (fixed seed) | | **Sensor readings** | Conveyor/crane motor telemetry: temperature, vibration, current, belt speed | 1,000 readings (900 normal + 100 anomalous) | Normal readings drawn from realistic operating ranges; anomalies simulate known failure signatures (elevated temp/vibration/current + reduced belt speed) | | **Retrieval eval set** | Hand-labelled (question → expected knowledge-base article) pairs | 17 pairs | Written by the developer to check retrieval accuracy | ## How each model works (function) | Model | Purpose | Algorithm | Input → Output | |---|---|---|---| | **Intent classifier** | Decide what kind of request a query is (maintenance, safety, inventory, order status, navigation, picking, system status, general FAQ) | TF-IDF + Logistic Regression | Free text → intent label + confidence | | **KB retriever** | Find the most relevant knowledge-base passage(s) for a query, to ground the LLM's answer | TF-IDF + cosine similarity | Free text → top-k ranked passages | | **LLM assistant** | Generate a natural-language answer grounded in the retrieved context | Hosted instruct LLM via an OpenAI-compatible API (Groq / custom / HF router), with fallback | Query + context → grounded answer (or an extractive fallback if the LLM is unavailable) | | **Inventory/order query** | Turn a question into a filtered table lookup | Regex slot extraction (SKU / order ID / zone) + intent routing | Free text → filtered inventory or orders table | | **Anomaly detector** | Flag abnormal equipment sensor readings before they cause a stoppage | Isolation Forest (unsupervised), StandardScaler | 4 sensor features → anomaly / normal + anomaly score | ## How to test this project 1. **AI Assistant** — try one of the example questions, or ask your own (e.g. *"The conveyor belt in Zone C is making noise"*). Check the metadata line under each answer to see the detected intent, which knowledge-base article(s) were retrieved, and whether the LLM or the fallback path answered. If the LLM path isn't working, open the **LLM connection diagnostics** accordion and click "Test LLM connection" for a precise error message. 2. **Inventory & Order Query** — try *"How many units of SKU-1042 are in Zone B?"* or *"What's the status of order #10007?"*, or browse the full synthetic tables in the accordion below the search box. 3. **Predictive Maintenance** — load one of the presets (Normal / Early bearing wear / Severe fault) or drag the sliders yourself, then click *"Check for anomaly"* to see the model's verdict and recommended action. 4. **Model Evaluation** — every chart here is generated on **held-out test data** by `build_artifacts.py`, not cherry-picked from a live run. Run that script yourself to reproduce every number from scratch. ## Tech stack - **UI / deployment:** [Gradio](https://gradio.app) on Hugging Face Spaces - **LLM:** any hosted model behind an **OpenAI-compatible chat API** — Groq's free tier (`GROQ_API_KEY`), a custom endpoint (`LLM_API_KEY` / `LLM_BASE_URL` / `LLM_MODEL`), or Hugging Face's inference router. Plain HTTP rather than a provider-specific SDK, so there's no provider list to go stale. It stops after one attempt on a 402/401 (every model would fail the same way) and falls back to a retrieval-only answer, so the public demo never just breaks. - **Retrieval:** TF-IDF + cosine similarity over a small hand-written warehouse-operations knowledge base (simple, fast, fully local RAG). - **Intent classification:** TF-IDF + Logistic Regression (scikit-learn) -- chosen deliberately over a heavier transformer classifier because it trains in under a second and comfortably reaches **{intent_eval.get('accuracy', 0):.0%} accuracy** on this task; right-sizing the model to the problem. - **Anomaly detection:** Isolation Forest (scikit-learn), trained unsupervised on scaled sensor features. - **Evaluation:** scikit-learn metrics + matplotlib, all computed by `build_artifacts.py` and saved as static artifacts the app loads at startup (fast, reproducible Space boot, self-healing if the deployed scikit-learn version ever drifts from the one used to train the models). ## Architecture ``` ┌───────────────────────┐ User query ───► │ Intent Classifier │ (TF-IDF + LogisticRegression) └──────────┬────────────┘ │ intent label ▼ ┌───────────────────────┐ │ KB Retriever (RAG) │ (TF-IDF cosine similarity) └──────────┬────────────┘ │ top-k passages ▼ ┌───────────────────────┐ │ Hosted LLM │ (OpenAI-compatible API) │ (or extractive │ │ fallback if offline) │ └──────────┬────────────┘ ▼ Grounded answer Sensor stream ───► StandardScaler ───► IsolationForest ───► anomaly / normal ``` ## Why this approach The value of AI in a warehouse-automation context isn't a flashy chatbot — it's **routing, grounding, and reliability**: correctly triaging a request, answering from real operational context instead of guessing, and flagging equipment problems before they cause downtime. Every component here was chosen to be as simple as it can be while still doing that job well and being honestly evaluated, rather than reaching for the biggest available model by default. ## Extended capabilities - **Multi-turn conversation** — the assistant keeps the last few turns, so follow-ups like *"what about Zone D instead?"* work. - **📷 Visual Inventory Scan** — runs YOLO on **real warehouse photos** (upload your own, or load an openly-licensed example). *Standard* mode (COCO-pretrained YOLOv8) finds people and vehicles but has no pallet/forklift/box class; *Find by name* mode (open-vocabulary YOLOE) searches for whatever you type, zero-shot. **Honest limit:** accuracy on real warehouse photos is *not measured* (no labelled warehouse images were available). A classical-CV baseline on synthetic racks, where ground truth is known, *is* measured: 100% on clean renders, degrading under simulated noise (see Model Evaluation). **Detector options considered for real photos** (latency = rough figures from a development CPU, after model load): | Option | Finds pallets / forklifts / boxes? | Weight & speed | Status | |---|---|---|---| | YOLOv8 (COCO) | **No** — no such classes; finds people & vehicles | 6–22 MB, ~75 ms | ✅ Shipped; run on a real photo | | YOLOE (open-vocabulary) | Yes, zero-shot via text prompts | 28 MB + ~600 MB text encoder, ~250 ms; first load ≈ 1–2 min | ✅ Shipped; run on a real photo (not on warehouse photos) | | YOLO-World (open-vocabulary) | Yes, zero-shot | text encoder hosted on a CDN I couldn't reach | ❌ Not shipped — couldn't verify it | | OWLv2 / Grounding DINO | Yes, zero-shot, often stronger | hundreds of MB, seconds per image on CPU; weights on the Hugging Face Hub, unreachable from my environment | ❌ Not shipped — couldn't verify it | | YOLO fine-tuned on a warehouse dataset | Best accuracy | needs licensed, labelled warehouse photos + training | ⏳ Recommended next step | | Classical CV (OpenCV) | Only clean synthetic racks | tiny | ✅ Kept as a *measured* baseline | - **Batch CSV scoring** — upload a CSV of sensor readings (a sample is provided) to score a whole shift's logs at once, and download the results. - **Slack alerting** — anomalies can trigger a Slack alert. Real if a `SLACK_WEBHOOK_URL` secret is set; otherwise a clearly-labelled preview that is *never* presented as sent. - **Bilingual UI** — switch the explanatory text between English and 日本語 with the selector at the top. (Charts and metric tables stay in English; the assistant answers in whatever language you ask in.) ## Limitations & next steps - All data here is **synthetic**, for portfolio/demo purposes — a production version would connect to real WMS/WCS APIs and historical sensor logs. - The intent set (8 classes) and knowledge base (17 articles) are intentionally small to keep the demo fast and auditable; both are easy to extend. - The anomaly detector uses 4 hand-picked features; a production system would likely use a richer multivariate sensor set and a supervised or semi-supervised model once labelled failure data is available. --- *Built as a portfolio/application project. Source code available on request or in the linked repository. Feedback welcome.* """ # ========================================================================== # GRADIO APP # ========================================================================== CUSTOM_CSS = """ #title-banner { text-align: center; margin-bottom: 0.5em; } .gradio-container { max-width: 1150px !important; margin: auto; } .data-badge { background: #eff6ff; border-left: 4px solid #3b82f6; border-radius: 6px; padding: 10px 14px; margin-bottom: 10px; font-size: 0.92em; line-height: 1.5; } .data-badge b { color: #1e3a8a; } """ # ========================================================================== # Bilingual (English / 日本語) static content # ========================================================================== # Static explanatory text (subtitle, per-tab data/function panels, About tab) # is switchable via the language selector. Dynamic content (charts, metric # tables, the LLM's answers) stays in English; the assistant itself will # answer in whatever language the question is asked in, since the hosted LLM # is multilingual. from src.knowledge_base import KNOWLEDGE_BASE _N_KB = len(KNOWLEDGE_BASE) I18N = { "subtitle": { "en": "
LLM-powered intralogistics copilot · intent routing · " "multi-turn RAG · predictive maintenance · computer vision
", "ja": "LLM搭載の物流オペレーション支援AI · 意図分類 · " "マルチターン対話(RAG) · 予知保全 · コンピュータビジョン
", }, "badge_assistant": { "en": "🧪 Source: hand-written by the developer — %d original knowledge-base articles on warehouse " "operations (AS/RS, AGV/AMR, WMS, sortation, picking, maintenance, safety, cross-docking, RFID, slotting, " "labor management…) plus ~480 template-generated example queries. Not scraped or sourced from any real company.src/data_generation.py, fixed seed) — "
f"{len(inventory_df)} synthetic SKU records and {len(orders_df)} synthetic orders. Not exported from any real company's WMS.src/data_generation.py、乱数シード固定)で生成した合成データ"
f"(SKU {len(inventory_df)}件、注文 {len(orders_df)}件)。実在企業のWMSから出力したものではありません。src/data_generation.py, fixed seed) — 1,000 synthetic "
"sensor readings (900 normal + 100 simulated-fault). Not logged from real equipment.SLACK_WEBHOOK_URL secret is set, "
"otherwise a clearly-labelled preview).",
"ja": "🧪 データの出典:開発者のコード(src/data_generation.py、乱数シード固定)で生成した合成センサーデータ1,000件"
"(正常900+故障模擬100)。実機のログではありません。SLACK_WEBHOOK_URL設定時は実送信、未設定時はプレビュー表示)。",
},
"badge_vision": {
"en": "🧪 Source: Real photos: anything you upload (processed in memory, not stored), plus three openly-licensed "
"example photos fetched at runtime from Wikimedia Commons with author and license shown — no third-party images are bundled "
"in this repo. Synthetic baseline: rack images generated by src/vision_model.py, where ground truth is known.src/vision_model.pyで生成した、正解が既知のラック画像。build_artifacts.py — fully reproducible, not cherry-picked.",
"ja": "🧪 以下の図表はすべて、build_artifacts.pyがホールドアウトしたテストデータで算出したものです。再現可能で、結果の取捨選択は行っていません。",
},
}
def about_ja() -> str:
return f"""
# 🏭 スマート倉庫AIアシスタント
**物流・倉庫自動化の運用を題材に、実務的なAI活用を示すポートフォリオプロジェクトです。**
AS/RS、コンベヤ・仕分け、AGV/AMR、WMS/WCSといった倉庫自動化の現場では、設備テレメトリ、取引データ、安全記録、現場からの問い合わせなど
大量の運用データが生まれます。本プロジェクトは、その上にAIレイヤーを載せる最小構成の実装例です。依頼を正しく振り分け、推測ではなく
根拠のある情報で回答し、設備の異常を早期に検出し、さらに**再現可能で誠実な評価**を行うことを重視しています。
## 機能一覧
| 機能 | タブ | 主な技術 |
|---|---|---|
| ナレッジに基づく**LLM対話アシスタント**(RAG/マルチターン会話対応) | AI Assistant | TF-IDF検索 + ホスト型LLM |
| **意図分類**による問い合わせの振り分け(8カテゴリ) | AI Assistant / Inventory | TF-IDF + ロジスティック回帰 |
| 在庫・注文の**自然文照会** | Inventory & Order Query | 正規表現によるスロット抽出 + 意図分類 |
| センサーデータの**予知保全**(1件判定/CSV一括判定/Slack通知) | Predictive Maintenance | Isolation Forest(教師なし)+ 較正済みしきい値 |
| **実写真の物体検出**(人・車両/名前指定でパレット等)+合成ラックのベースライン | Visual Inventory Scan | YOLOv8 / YOLOE + 古典的画像処理 |
| 全コンポーネントの**再現可能な評価** | Model Evaluation | scikit-learn指標、ホールドアウト検証 |
## データセットの概要
**本プロジェクトのデータはすべて、自作コードによる合成データです。**実在企業のシステムからの取得・流用は一切ありません。
プライバシーやライセンスの懸念なく、誰でも再現・共有できます。
| データ | 内容 | 件数 | 生成方法 |
|---|---|---|---|
| ナレッジベース | 倉庫自動化の一般的概念を解説した手書き記事 | {_N_KB}記事 | 本プロジェクトのために作成 |
| 意図分類クエリ | 8カテゴリの問い合わせ例 | 約480件 | テンプレートにSKU・ゾーン・設備IDなどをランダムに埋め込み |
| 在庫テーブル | SKU・カテゴリ・ゾーン・在庫数・発注点・単価 | 60 SKU | 現実的な範囲でランダム生成(シード固定) |
| 注文テーブル | ステータス・明細数・優先度・ゾーン | 80件 | 現実的な範囲でランダム生成(シード固定) |
| センサーデータ | 温度・振動・電流・ベルト速度 | 1,000件(正常900/異常100) | 正常は運転範囲から、異常は故障パターンを模して生成 |
| ラック画像 | 箱の配置が既知の合成画像 | 随時生成 | PILで描画。ノイズ強度を調整可能 |
| 検索評価セット | 質問と正解記事のペア | {_N_KB}組 | 検索精度確認のため手作業で作成 |
## 各モデルの役割
| モデル | 目的 | アルゴリズム | 入力 → 出力 |
|---|---|---|---|
| 意図分類器 | 問い合わせの種類を判定 | TF-IDF + ロジスティック回帰 | 文章 → 意図ラベル + 確信度 |
| 検索(RAG) | 回答の根拠となる記事を取得 | TF-IDF + コサイン類似度 | 文章 → 関連記事の上位k件 |
| LLMアシスタント | 根拠に基づく自然な回答を生成 | ホスト型LLM(複数モデルのフォールバック) | 質問 + 根拠 + 会話履歴 → 回答 |
| 在庫・注文照会 | 質問をテーブル検索に変換 | 正規表現 + 意図分類 | 文章 → 絞り込み結果 |
| 異常検知 | 設備の異常な計測値を早期に検出 | Isolation Forest + 較正済みしきい値 | 4つのセンサー値 → 正常/異常 + スコア |
| 画像検出器 | 実写真から物体を検出 | YOLOv8(COCO)/YOLOE(名前指定)、ベースラインは古典的画像処理 | 画像 → 検出枠 + クラス別件数 |
## 試し方
1. **AI Assistant** — 例の質問を選ぶか自由に入力します。続けて「Zone Dの場合は?」などと尋ねると、直前の文脈を踏まえて回答します。
2. **Inventory & Order Query** — 「SKU-1042 は Zone B に何個ある?」「注文 #10007 の状況は?」などを試します。
3. **Predictive Maintenance** — プリセットまたはスライダーで1件を判定します。「CSV一括判定」でサンプルCSVをダウンロードし、そのままアップロードして一括判定も試せます。
4. **Visual Inventory Scan** — 実写真をアップロードするか例示写真を読み込み、「名前で検索」で *pallet, forklift* などを探します(実写真での精度は未測定)。合成ラックのタブでは、ノイズを上げて検出が難しくなる様子を確認できます。
5. **Model Evaluation** — すべての図表はテストデータで算出しており、`build_artifacts.py`を再実行すれば同じ数値を再現できます。
## 評価の信頼性について
開発中に2件のデータリーク(意図分類での学習/テスト間の文面重複、異常検知でのしきい値への正解率の混入)を発見・修正しました。
修正後の指標は、意図分類 正解率 {intent_eval.get('accuracy', 0):.1%}、異常検知 F1 {anomaly_eval.get('f1', 0):.1%} です。
合成ラックの古典的画像処理はノイズなしでは100%となるため、ノイズを加えた条件(標準ノイズ0.4で完全一致率 {vision_eval.get('exact_match_rate', 0):.0%})を報告しています。実写真に対するYOLOの精度は未測定です。
## 制約と今後の課題
- データはすべて**合成**です。実運用ではWMS/WCSのAPIや実際のセンサーログに接続します。
- 実写真向けのYOLOはゼロショット/汎用モデルであり、倉庫専用データでの微調整・評価は未実施です。精度を保証するには、ラベル付き倉庫画像(約100枚〜)でmAPを測定する必要があります。
- Slack通知は`SLACK_WEBHOOK_URL`未設定の場合はプレビューのみで、実際には送信されません。
"""
def data_badge(html: str) -> str:
return f'