""" llm_client.py ------------- Answers warehouse-operations questions with a hosted LLM, grounded in context retrieved from the local knowledge base (simple RAG), with multi-turn memory. Why this talks plain HTTP instead of `huggingface_hub.InferenceClient` --------------------------------------------------------------------- An earlier version used InferenceClient(provider="auto"). That broke twice in the wild: (1) Hugging Face's router started assigning models to providers ('featherless-ai', 'deepinfra') that the pinned client version didn't know, raising "Provider ... not supported"; (2) the account had no Inference Providers credits, giving HTTP 402. Everything modern (Groq, OpenRouter, Together, Cerebras, HF's own router, ...) exposes the same OpenAI-compatible `/chat/completions` protocol, so this module speaks that directly. There is no provider enum to go stale, and any such endpoint can be used by setting env vars / Space secrets. Backends, tried in this order (first success wins) -------------------------------------------------- 1. Custom OpenAI-compatible endpoint: LLM_API_KEY + LLM_BASE_URL + LLM_MODEL or just `GROQ_API_KEY` (Groq has a free plan; base URL and a default model are filled in; override the model with LLM_MODEL). 2. Hugging Face Inference Providers router (https://router.huggingface.co/v1), using the HF token. NOTE: free HF accounts get very little monthly credit; when it runs out the router answers 402 and this backend is skipped. 3. Extractive fallback: the retrieved knowledge-base passages are shown directly, clearly labelled -- the demo never breaks. Failure handling: a 402/401/403 means *every* model on that backend will fail the same way, so the backend is abandoned after one attempt instead of hammering six models. Error text never includes API keys. """ import os import re import time from dataclasses import dataclass, field from typing import List, Optional, Tuple import requests from src.retriever import KBRetriever, RetrievedDoc # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- GROQ_BASE_URL = "https://api.groq.com/openai/v1" GROQ_DEFAULT_MODEL = "llama-3.1-8b-instant" HF_ROUTER_BASE_URL = os.environ.get("HF_ROUTER_BASE_URL", "https://router.huggingface.co/v1") # Models tried on the Hugging Face router, in order (LLM_MODEL_ID goes first if set). MODEL_CANDIDATES = [ "Qwen/Qwen2.5-7B-Instruct", "meta-llama/Llama-3.1-8B-Instruct", "meta-llama/Llama-3.2-3B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3", ] _env_model = os.environ.get("LLM_MODEL_ID") if _env_model: MODEL_CANDIDATES = [_env_model] + [m for m in MODEL_CANDIDATES if m != _env_model] DEFAULT_MODEL_ID = MODEL_CANDIDATES[0] # The HF token is normally `HF_TOKEN`; a few other names are accepted too. TOKEN_ENV_VAR_CANDIDATES = [ "HF_TOKEN", "Smart_Warehouse", "HUGGINGFACE_TOKEN", "HUGGINGFACEHUB_API_TOKEN", "HF_API_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HF_ACCESS_TOKEN", ] REQUEST_TIMEOUT_S = 40 TOTAL_BUDGET_S = 75 # stop trying more candidates after this long (UI shouldn't hang) MAX_HISTORY_TURNS = 4 # prior (user, assistant) pairs carried into the call SYSTEM_PROMPT = ( "You are the Smart Warehouse AI Assistant, a helpful operations copilot " "for a large automated distribution center (conveyors, AS/RS, AGVs/AMRs, " "sortation, and a WMS). Answer concisely and practically, in the tone of " "an experienced warehouse operations engineer. Use the provided CONTEXT " "when relevant, and say so plainly if the question is outside the " "context. Prefer short paragraphs or bullet points over long prose. " "Reply in the same language the user wrote in." ) def _get_hf_token() -> Optional[str]: for var in TOKEN_ENV_VAR_CANDIDATES: val = os.environ.get(var) if val: return val return None @dataclass class Backend: label: str base_url: str api_key: str models: List[str] def get_backends() -> List[Backend]: """Backends configured via env vars / Space secrets, in priority order.""" backends: List[Backend] = [] key, url, model = os.environ.get("LLM_API_KEY"), os.environ.get("LLM_BASE_URL"), os.environ.get("LLM_MODEL") if key and url: host = re.sub(r"^https?://", "", url).split("/")[0] backends.append(Backend(f"custom ({host})", url.rstrip("/"), key, [model] if model else [])) groq_key = os.environ.get("GROQ_API_KEY") if groq_key: backends.append(Backend("Groq", GROQ_BASE_URL, groq_key, [model or GROQ_DEFAULT_MODEL])) hf = _get_hf_token() if hf: backends.append(Backend("Hugging Face router", HF_ROUTER_BASE_URL.rstrip("/"), hf, list(MODEL_CANDIDATES))) return backends def llm_configured() -> bool: return bool(get_backends()) # --------------------------------------------------------------------------- # Wire protocol (OpenAI-compatible /chat/completions) # --------------------------------------------------------------------------- class LLMError(Exception): """kind: credits | auth | rate | not_found | server | network | parse | empty""" def __init__(self, kind: str, message: str, status: Optional[int] = None): super().__init__(message) self.kind, self.status = kind, status # Failures after which trying more models on the SAME backend is pointless. BACKEND_FATAL = {"credits", "auth"} def _scrub(text: str, secret: str) -> str: text = text.replace(secret, "***") if secret else text return re.sub(r"(hf_|gsk_|sk-)[A-Za-z0-9_\-]{8,}", r"\1***", text) def _error_message(resp: requests.Response) -> str: try: data = resp.json() err = data.get("error", data) if isinstance(err, dict): return str(err.get("message") or err)[:240] return str(err)[:240] except Exception: # noqa: BLE001 return (resp.text or "")[:240].strip() def _chat(backend: Backend, model: str, messages: list, max_tokens: int, temperature: float = 0.3) -> str: try: resp = requests.post( f"{backend.base_url}/chat/completions", headers={"Authorization": f"Bearer {backend.api_key}", "Content-Type": "application/json"}, json={"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature}, timeout=REQUEST_TIMEOUT_S, ) except requests.exceptions.Timeout: raise LLMError("network", f"timed out after {REQUEST_TIMEOUT_S}s") except requests.exceptions.RequestException as e: raise LLMError("network", _scrub(f"{type(e).__name__}: {e}", backend.api_key)[:200]) s = resp.status_code if s == 200: try: content = resp.json()["choices"][0]["message"]["content"] except Exception: # noqa: BLE001 raise LLMError("parse", "response was not a valid chat completion", s) if not content or not str(content).strip(): raise LLMError("empty", "model returned an empty answer", s) return str(content).strip() detail = _scrub(_error_message(resp), backend.api_key) if s == 402: raise LLMError("credits", detail or "payment required", s) if s in (401, 403): raise LLMError("auth", detail or "unauthorized", s) if s == 429: raise LLMError("rate", detail or "rate limited", s) if s in (400, 404, 410, 422): raise LLMError("not_found", detail or "model/request not accepted", s) raise LLMError("server", f"HTTP {s}: {detail}", s) _HINTS = { "credits": ("402 Payment Required — this account has no inference credits " "(Hugging Face free accounts get almost none). Use a free-tier provider instead: " "set a `GROQ_API_KEY` secret (free key at console.groq.com), or add credits."), "auth": "key rejected (401/403) — check the secret's value and permissions.", "rate": "rate limited (429) — the free tier's per-minute/day limit was hit; wait and retry.", } def _explain(backend: Backend, model: str, e: LLMError) -> str: base = f"{backend.label} · {model or '(no model set)'}: {e.kind}" hint = _HINTS.get(e.kind) return f"{base} — {hint}" if hint else f"{base} — {e}" # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @dataclass class AssistantResponse: answer: str used_llm: bool sources: List[RetrievedDoc] latency_s: float model_id: str debug_errors: List[str] = field(default_factory=list) # filled when used_llm is False def _extractive_fallback(query: str, sources: List[RetrievedDoc]) -> str: """Deterministic answer used when no LLM is configured / reachable.""" if not sources: return ("I don't have grounded context for that yet. Try asking about inventory, order status, " "equipment maintenance, AGV routing, picking strategy, safety incidents, or general " "warehouse automation concepts.") bullets = "\n".join(f"- **{s.title}**: {s.text}" for s in sources) return (f"(Showing retrieved knowledge instead of an LLM-generated answer -- see the diagnostics below.)\n\n" f"Based on **{sources[0].title}**, here's the relevant information:\n\n{bullets}") def _clean_history(history: Optional[List[dict]]) -> List[dict]: """Normalize Gradio's messages-format history: strip our own metadata footer from assistant turns and keep only the last MAX_HISTORY_TURNS pairs (bounded prompt).""" if not history: return [] cleaned = [] for turn in history: role, content = turn.get("role"), turn.get("content", "") if not isinstance(content, str): continue if role == "assistant" and "\n\n---\n" in content: content = content.split("\n\n---\n")[0] if role in ("user", "assistant") and content.strip(): cleaned.append({"role": role, "content": content.strip()}) return cleaned[-MAX_HISTORY_TURNS * 2:] def answer_query( query: str, retriever: KBRetriever, k: int = 2, max_tokens: int = 350, history: Optional[List[dict]] = None, ) -> AssistantResponse: """`history`: prior turns (Gradio messages format) so follow-ups like "what about Zone D instead?" work. Omit for single-turn.""" start = time.time() sources = retriever.retrieve(query, k=k) context_block = "\n\n".join(f"[{s.title}]\n{s.text}" for s in sources) messages = [{"role": "system", "content": SYSTEM_PROMPT}] messages.extend(_clean_history(history)) messages.append({"role": "user", "content": f"CONTEXT:\n{context_block}\n\nQUESTION: {query}"}) backends = get_backends() errors: List[str] = [] if not backends: errors.append("No LLM configured. Set a `GROQ_API_KEY` secret (free at console.groq.com), " "or `LLM_API_KEY` + `LLM_BASE_URL` + `LLM_MODEL` for any OpenAI-compatible API, " "or an `HF_TOKEN` (needs Hugging Face inference credits).") for backend in backends: if not backend.models: errors.append(f"{backend.label}: LLM_MODEL is not set.") continue for model in backend.models: if time.time() - start > TOTAL_BUDGET_S: errors.append(f"Stopped after {TOTAL_BUDGET_S}s without a successful answer.") break try: text = _chat(backend, model, messages, max_tokens) return AssistantResponse(text, True, sources, time.time() - start, f"{backend.label}: {model}") except LLMError as e: errors.append(_explain(backend, model, e)) if e.kind in BACKEND_FATAL: break # every other model on this backend would fail identically return AssistantResponse(_extractive_fallback(query, sources), False, sources, time.time() - start, "extractive-fallback", debug_errors=errors) def test_connection(retriever: Optional[KBRetriever] = None) -> AssistantResponse: """One canned query through the full pipeline -- used by the diagnostics button.""" retriever = retriever or KBRetriever() return answer_query("What is a WMS?", retriever, k=1, max_tokens=60)