Spaces:
Running on Zero
Running on Zero
Download src/llm_client.py from Pro-Coder/SmartWareHouseAI: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/spaces/Pro-Coder/SmartWareHouseAI/resolve/main/src/llm_client.py
- Command line
-
hf download hf://spaces/Pro-Coder/SmartWareHouseAI/src/llm_client.py
-
curl -L -o llm_client.py https://huggingface.co/spaces/Pro-Coder/SmartWareHouseAI/resolve/main/src/llm_client.py
12.5 kB
| """ | |
| llm_client.py | |
| ------------- | |
| Answers warehouse-operations questions with a hosted LLM, grounded in context | |
| retrieved from the local knowledge base (simple RAG), with multi-turn memory. | |
| Why this talks plain HTTP instead of `huggingface_hub.InferenceClient` | |
| --------------------------------------------------------------------- | |
| An earlier version used InferenceClient(provider="auto"). That broke twice in | |
| the wild: (1) Hugging Face's router started assigning models to providers | |
| ('featherless-ai', 'deepinfra') that the pinned client version didn't know, | |
| raising "Provider ... not supported"; (2) the account had no Inference | |
| Providers credits, giving HTTP 402. Everything modern (Groq, OpenRouter, | |
| Together, Cerebras, HF's own router, ...) exposes the same OpenAI-compatible | |
| `/chat/completions` protocol, so this module speaks that directly. There is | |
| no provider enum to go stale, and any such endpoint can be used by setting | |
| env vars / Space secrets. | |
| Backends, tried in this order (first success wins) | |
| -------------------------------------------------- | |
| 1. Custom OpenAI-compatible endpoint: | |
| LLM_API_KEY + LLM_BASE_URL + LLM_MODEL | |
| or just `GROQ_API_KEY` (Groq has a free plan; base URL and a default model | |
| are filled in; override the model with LLM_MODEL). | |
| 2. Hugging Face Inference Providers router (https://router.huggingface.co/v1), | |
| using the HF token. NOTE: free HF accounts get very little monthly credit; | |
| when it runs out the router answers 402 and this backend is skipped. | |
| 3. Extractive fallback: the retrieved knowledge-base passages are shown | |
| directly, clearly labelled -- the demo never breaks. | |
| Failure handling: a 402/401/403 means *every* model on that backend will fail | |
| the same way, so the backend is abandoned after one attempt instead of | |
| hammering six models. Error text never includes API keys. | |
| """ | |
| import os | |
| import re | |
| import time | |
| from dataclasses import dataclass, field | |
| from typing import List, Optional, Tuple | |
| import requests | |
| from src.retriever import KBRetriever, RetrievedDoc | |
| # --------------------------------------------------------------------------- | |
| # Configuration | |
| # --------------------------------------------------------------------------- | |
| GROQ_BASE_URL = "https://api.groq.com/openai/v1" | |
| GROQ_DEFAULT_MODEL = "llama-3.1-8b-instant" | |
| HF_ROUTER_BASE_URL = os.environ.get("HF_ROUTER_BASE_URL", "https://router.huggingface.co/v1") | |
| # Models tried on the Hugging Face router, in order (LLM_MODEL_ID goes first if set). | |
| MODEL_CANDIDATES = [ | |
| "Qwen/Qwen2.5-7B-Instruct", | |
| "meta-llama/Llama-3.1-8B-Instruct", | |
| "meta-llama/Llama-3.2-3B-Instruct", | |
| "mistralai/Mistral-7B-Instruct-v0.3", | |
| ] | |
| _env_model = os.environ.get("LLM_MODEL_ID") | |
| if _env_model: | |
| MODEL_CANDIDATES = [_env_model] + [m for m in MODEL_CANDIDATES if m != _env_model] | |
| DEFAULT_MODEL_ID = MODEL_CANDIDATES[0] | |
| # The HF token is normally `HF_TOKEN`; a few other names are accepted too. | |
| TOKEN_ENV_VAR_CANDIDATES = [ | |
| "HF_TOKEN", "Smart_Warehouse", "HUGGINGFACE_TOKEN", "HUGGINGFACEHUB_API_TOKEN", | |
| "HF_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_ACCESS_TOKEN", | |
| ] | |
| REQUEST_TIMEOUT_S = 40 | |
| TOTAL_BUDGET_S = 75 # stop trying more candidates after this long (UI shouldn't hang) | |
| MAX_HISTORY_TURNS = 4 # prior (user, assistant) pairs carried into the call | |
| SYSTEM_PROMPT = ( | |
| "You are the Smart Warehouse AI Assistant, a helpful operations copilot " | |
| "for a large automated distribution center (conveyors, AS/RS, AGVs/AMRs, " | |
| "sortation, and a WMS). Answer concisely and practically, in the tone of " | |
| "an experienced warehouse operations engineer. Use the provided CONTEXT " | |
| "when relevant, and say so plainly if the question is outside the " | |
| "context. Prefer short paragraphs or bullet points over long prose. " | |
| "Reply in the same language the user wrote in." | |
| ) | |
| def _get_hf_token() -> Optional[str]: | |
| for var in TOKEN_ENV_VAR_CANDIDATES: | |
| val = os.environ.get(var) | |
| if val: | |
| return val | |
| return None | |
| class Backend: | |
| label: str | |
| base_url: str | |
| api_key: str | |
| models: List[str] | |
| def get_backends() -> List[Backend]: | |
| """Backends configured via env vars / Space secrets, in priority order.""" | |
| backends: List[Backend] = [] | |
| key, url, model = os.environ.get("LLM_API_KEY"), os.environ.get("LLM_BASE_URL"), os.environ.get("LLM_MODEL") | |
| if key and url: | |
| host = re.sub(r"^https?://", "", url).split("/")[0] | |
| backends.append(Backend(f"custom ({host})", url.rstrip("/"), key, [model] if model else [])) | |
| groq_key = os.environ.get("GROQ_API_KEY") | |
| if groq_key: | |
| backends.append(Backend("Groq", GROQ_BASE_URL, groq_key, [model or GROQ_DEFAULT_MODEL])) | |
| hf = _get_hf_token() | |
| if hf: | |
| backends.append(Backend("Hugging Face router", HF_ROUTER_BASE_URL.rstrip("/"), hf, list(MODEL_CANDIDATES))) | |
| return backends | |
| def llm_configured() -> bool: | |
| return bool(get_backends()) | |
| # --------------------------------------------------------------------------- | |
| # Wire protocol (OpenAI-compatible /chat/completions) | |
| # --------------------------------------------------------------------------- | |
| class LLMError(Exception): | |
| """kind: credits | auth | rate | not_found | server | network | parse | empty""" | |
| def __init__(self, kind: str, message: str, status: Optional[int] = None): | |
| super().__init__(message) | |
| self.kind, self.status = kind, status | |
| # Failures after which trying more models on the SAME backend is pointless. | |
| BACKEND_FATAL = {"credits", "auth"} | |
| def _scrub(text: str, secret: str) -> str: | |
| text = text.replace(secret, "***") if secret else text | |
| return re.sub(r"(hf_|gsk_|sk-)[A-Za-z0-9_\-]{8,}", r"\1***", text) | |
| def _error_message(resp: requests.Response) -> str: | |
| try: | |
| data = resp.json() | |
| err = data.get("error", data) | |
| if isinstance(err, dict): | |
| return str(err.get("message") or err)[:240] | |
| return str(err)[:240] | |
| except Exception: # noqa: BLE001 | |
| return (resp.text or "")[:240].strip() | |
| def _chat(backend: Backend, model: str, messages: list, max_tokens: int, temperature: float = 0.3) -> str: | |
| try: | |
| resp = requests.post( | |
| f"{backend.base_url}/chat/completions", | |
| headers={"Authorization": f"Bearer {backend.api_key}", "Content-Type": "application/json"}, | |
| json={"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature}, | |
| timeout=REQUEST_TIMEOUT_S, | |
| ) | |
| except requests.exceptions.Timeout: | |
| raise LLMError("network", f"timed out after {REQUEST_TIMEOUT_S}s") | |
| except requests.exceptions.RequestException as e: | |
| raise LLMError("network", _scrub(f"{type(e).__name__}: {e}", backend.api_key)[:200]) | |
| s = resp.status_code | |
| if s == 200: | |
| try: | |
| content = resp.json()["choices"][0]["message"]["content"] | |
| except Exception: # noqa: BLE001 | |
| raise LLMError("parse", "response was not a valid chat completion", s) | |
| if not content or not str(content).strip(): | |
| raise LLMError("empty", "model returned an empty answer", s) | |
| return str(content).strip() | |
| detail = _scrub(_error_message(resp), backend.api_key) | |
| if s == 402: | |
| raise LLMError("credits", detail or "payment required", s) | |
| if s in (401, 403): | |
| raise LLMError("auth", detail or "unauthorized", s) | |
| if s == 429: | |
| raise LLMError("rate", detail or "rate limited", s) | |
| if s in (400, 404, 410, 422): | |
| raise LLMError("not_found", detail or "model/request not accepted", s) | |
| raise LLMError("server", f"HTTP {s}: {detail}", s) | |
| _HINTS = { | |
| "credits": ("402 Payment Required — this account has no inference credits " | |
| "(Hugging Face free accounts get almost none). Use a free-tier provider instead: " | |
| "set a `GROQ_API_KEY` secret (free key at console.groq.com), or add credits."), | |
| "auth": "key rejected (401/403) — check the secret's value and permissions.", | |
| "rate": "rate limited (429) — the free tier's per-minute/day limit was hit; wait and retry.", | |
| } | |
| def _explain(backend: Backend, model: str, e: LLMError) -> str: | |
| base = f"{backend.label} · {model or '(no model set)'}: {e.kind}" | |
| hint = _HINTS.get(e.kind) | |
| return f"{base} — {hint}" if hint else f"{base} — {e}" | |
| # --------------------------------------------------------------------------- | |
| # Public API | |
| # --------------------------------------------------------------------------- | |
| class AssistantResponse: | |
| answer: str | |
| used_llm: bool | |
| sources: List[RetrievedDoc] | |
| latency_s: float | |
| model_id: str | |
| debug_errors: List[str] = field(default_factory=list) # filled when used_llm is False | |
| def _extractive_fallback(query: str, sources: List[RetrievedDoc]) -> str: | |
| """Deterministic answer used when no LLM is configured / reachable.""" | |
| if not sources: | |
| return ("I don't have grounded context for that yet. Try asking about inventory, order status, " | |
| "equipment maintenance, AGV routing, picking strategy, safety incidents, or general " | |
| "warehouse automation concepts.") | |
| bullets = "\n".join(f"- **{s.title}**: {s.text}" for s in sources) | |
| return (f"(Showing retrieved knowledge instead of an LLM-generated answer -- see the diagnostics below.)\n\n" | |
| f"Based on **{sources[0].title}**, here's the relevant information:\n\n{bullets}") | |
| def _clean_history(history: Optional[List[dict]]) -> List[dict]: | |
| """Normalize Gradio's messages-format history: strip our own metadata footer from | |
| assistant turns and keep only the last MAX_HISTORY_TURNS pairs (bounded prompt).""" | |
| if not history: | |
| return [] | |
| cleaned = [] | |
| for turn in history: | |
| role, content = turn.get("role"), turn.get("content", "") | |
| if not isinstance(content, str): | |
| continue | |
| if role == "assistant" and "\n\n---\n" in content: | |
| content = content.split("\n\n---\n")[0] | |
| if role in ("user", "assistant") and content.strip(): | |
| cleaned.append({"role": role, "content": content.strip()}) | |
| return cleaned[-MAX_HISTORY_TURNS * 2:] | |
| def answer_query( | |
| query: str, | |
| retriever: KBRetriever, | |
| k: int = 2, | |
| max_tokens: int = 350, | |
| history: Optional[List[dict]] = None, | |
| ) -> AssistantResponse: | |
| """`history`: prior turns (Gradio messages format) so follow-ups like | |
| "what about Zone D instead?" work. Omit for single-turn.""" | |
| start = time.time() | |
| sources = retriever.retrieve(query, k=k) | |
| context_block = "\n\n".join(f"[{s.title}]\n{s.text}" for s in sources) | |
| messages = [{"role": "system", "content": SYSTEM_PROMPT}] | |
| messages.extend(_clean_history(history)) | |
| messages.append({"role": "user", "content": f"CONTEXT:\n{context_block}\n\nQUESTION: {query}"}) | |
| backends = get_backends() | |
| errors: List[str] = [] | |
| if not backends: | |
| errors.append("No LLM configured. Set a `GROQ_API_KEY` secret (free at console.groq.com), " | |
| "or `LLM_API_KEY` + `LLM_BASE_URL` + `LLM_MODEL` for any OpenAI-compatible API, " | |
| "or an `HF_TOKEN` (needs Hugging Face inference credits).") | |
| for backend in backends: | |
| if not backend.models: | |
| errors.append(f"{backend.label}: LLM_MODEL is not set.") | |
| continue | |
| for model in backend.models: | |
| if time.time() - start > TOTAL_BUDGET_S: | |
| errors.append(f"Stopped after {TOTAL_BUDGET_S}s without a successful answer.") | |
| break | |
| try: | |
| text = _chat(backend, model, messages, max_tokens) | |
| return AssistantResponse(text, True, sources, time.time() - start, f"{backend.label}: {model}") | |
| except LLMError as e: | |
| errors.append(_explain(backend, model, e)) | |
| if e.kind in BACKEND_FATAL: | |
| break # every other model on this backend would fail identically | |
| return AssistantResponse(_extractive_fallback(query, sources), False, sources, | |
| time.time() - start, "extractive-fallback", debug_errors=errors) | |
| def test_connection(retriever: Optional[KBRetriever] = None) -> AssistantResponse: | |
| """One canned query through the full pipeline -- used by the diagnostics button.""" | |
| retriever = retriever or KBRetriever() | |
| return answer_query("What is a WMS?", retriever, k=1, max_tokens=60) | |