diff --git a/projects/ai/artifacts/ollama-toolcall-test.py b/projects/ai/artifacts/ollama-toolcall-test.py new file mode 100644 index 0000000..a2e1861 --- /dev/null +++ b/projects/ai/artifacts/ollama-toolcall-test.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Functional tool-calling test suite for local Ollama models (<10 GB, tools capability). + +10 scenarios per model, increasing difficulty, English-only prompts. + + S1 direct call explicit "use the tool", simple city arg + S2 implied call no explicit tool instruction, model must infer + S3 tool selection 3 tools available, must pick get_weather + S4 context resolution city referenced indirectly ("there") from context + S5 multiple parameters city + unit, unit must be extracted from request + S6 no-tool restraint answerable without tools -> must NOT call + S7 multi-call two cities compared -> both calls expected + S8 result reasoning use result to answer a yes/no derived question + S9 argument fidelity city with diacritics must be preserved + S10 distractor selection 5 tools, must pick the non-obvious get_stock_price + +Per scenario: PHASE 1 (emit valid tool call) + PHASE 2 (use the returned result). +Verdicts: PASS = 1, PARTIAL = 0.5, FAIL = 0. Prints each scenario result +immediately (unbuffered). Usage: + + PYTHONUNBUFFERED=1 uv run ollama-toolcall-test.py +""" +import json +import sys +import time +import unicodedata +import urllib.request + +BASE = "http://nvidia.hell:11434" +TIMEOUT = 120 # per request; abort the model run if exceeded + +MODELS = [ + "lfm2.5-thinking", # 0.7 + "deepseek-v3", # 2.0 + "llama3.2", # 2.0 + "granite4", # 2.1 + "phi4-mini", # 2.5 + "nemotron-mini", # 2.7 + "phi4-mini-reasoning", # 3.2 + "mistral", # 4.1 + "granite4:tiny-h", # 4.2 + "granite3.3", # 4.9 + "aya-expanse", # 5.1 + "qwen3", # 5.2 + "granite4.1:8b", # 5.3 + "gemma4-uncensored", # 5.3 + "ornith", # 5.6 + "qwen3.5-uncensored", # 5.6 + "ministral-3", # 6.0 + "qwen3.5:9b", # 6.6 + "qwen3.5", # 6.6 + "mistral-nemo", # 7.1 +] + + +def strip_accents(s): + return "".join(c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn") + + +def trunc(s, n=200): + s = (s or "").replace("\n", " ").strip() + return s[:n] + ("…" if len(s) > n else "") + + +LAST_TIMING = {} + + +def chat(payload, timeout=TIMEOUT): + req = urllib.request.Request( + f"{BASE}/api/chat", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + +def fdef(name, description, properties, required): + return {"type": "function", "function": { + "name": name, "description": description, + "parameters": {"type": "object", "properties": properties, "required": required}, + }} + + +W_CITY = {"city": {"type": "string", "description": "City name"}} + + +def weather_tool(unit=False): + props = dict(W_CITY) + if unit: + props["unit"] = {"type": "string", "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit for the result"} + return fdef("get_weather", "Get the current weather for a city", props, ["city"]) + + +def get_time_tool(): + return fdef("get_time", "Get the current local time in a timezone", + {"timezone": {"type": "string", "description": "IANA timezone, e.g. Europe/Prague"}}, ["timezone"]) + + +def get_currency_tool(): + return fdef("get_currency_rate", "Get the current exchange rate between two currencies", + {"from": {"type": "string"}, "to": {"type": "string"}}, ["from", "to"]) + + +def get_stock_tool(): + return fdef("get_stock_price", "Get the current stock price for a company", + {"symbol": {"type": "string", "description": "Stock ticker symbol, e.g. GOOG"}}, ["symbol"]) + + +def send_email_tool(): + return fdef("send_email", "Send an email to someone", + {"to": {"type": "string", "description": "Recipient email"}, + "subject": {"type": "string"}, "body": {"type": "string"}}, ["to"]) + + +def translate_tool(): + return fdef("translate_text", "Translate a text to another language", + {"text": {"type": "string"}, "target_language": {"type": "string"}}, ["text"]) + + +def tool_result(city, **kw): + d = {"city": city} + d.update(kw) + return json.dumps(d) + + +SCENARIOS = [ + { # 1 easy: explicit instruction, trivial extraction + "id": "S1", "name": "direct call", + "tools": [weather_tool()], + "messages": [{"role": "user", "content": "What is the current weather in Brno right now? Use the tool."}], + "expect_tool": "get_weather", "expect_args": {"city": "brno"}, + "phase2": True, + "result": tool_result("Brno", temperature_c=18, condition="partly cloudy", wind_kmh=7), + "answer_any": ["18"], + }, + { # 2: no "use the tool" hint at all + "id": "S2", "name": "implied call", + "tools": [weather_tool()], + "messages": [{"role": "user", "content": "I'm about to head out. Can you check what the weather's like in Prague?"}], + "expect_tool": "get_weather", "expect_args": {"city": "prague"}, + "phase2": True, + "result": tool_result("Prague", temperature_c=14, condition="overcast", wind_kmh=10), + "answer_any": ["14"], + }, + { # 3: pick the right tool out of 3 + "id": "S3", "name": "tool selection", + "tools": [weather_tool(), get_time_tool(), get_currency_tool()], + "messages": [{"role": "user", "content": "Should I take an umbrella when going to work in Ostrava today?"}], + "expect_tool": "get_weather", "expect_args": {"city": "ostrava"}, + "phase2": True, + "result": tool_result("Ostrava", temperature_c=12, condition="rain", wind_kmh=20, precipitation_prob="85%"), + "answer_any": ["umbrella", "rain"], + }, + { # 4: city only implied by conversation context + "id": "S4", "name": "context resolution", + "tools": [weather_tool()], + "messages": [ + {"role": "user", "content": "My sister lives in Pardubice."}, + {"role": "assistant", "content": "Nice! Pardubice is a lovely city. Is there anything I can help you with?"}, + {"role": "user", "content": "What's the weather like there right now?"}, + ], + "expect_tool": "get_weather", "expect_args": {"city": "pardubice"}, + "phase2": True, + "result": tool_result("Pardubice", temperature_c=21, condition="clear", wind_kmh=5), + "answer_any": ["21"], + }, + { # 5: two parameters, second one from wording + "id": "S5", "name": "multiple parameters", + "tools": [weather_tool(unit=True)], + "messages": [{"role": "user", "content": "What's the weather in Vienna? I'd like it in Fahrenheit."}], + "expect_tool": "get_weather", "expect_args": {"city": "vienna", "unit": "fahr"}, + "phase2": True, + "result": json.dumps({"city": "Vienna", "temperature_f": 64, "condition": "sunny", "humidity": "35%"}), + "answer_any": ["64"], + }, + { # 6: must NOT call any tool + "id": "S6", "name": "no-tool restraint", + "tools": [weather_tool()], + "messages": [{"role": "user", "content": "What is the capital of Germany?"}], + "no_call": True, "phase2": False, + "answer_any": ["berlin"], + }, + { # 7: two tool calls in one answer + "id": "S7", "name": "multi-call", + "tools": [weather_tool()], + "messages": [{"role": "user", "content": "Compare the current weather in Brno and Olomouc."}], + "multi_call": ["brno", "olomouc"], + "phase2": False, + }, + { # 8: answer a derived yes/no question from the result + "id": "S8", "name": "result reasoning", + "tools": [weather_tool()], + "messages": [{"role": "user", "content": "Is it too hot for a run in Plzen right now?"}], + "expect_tool": "get_weather", "expect_args": {"city": "plzen"}, + "phase2": True, + "result": tool_result("Plzen", temperature_c=31, condition="sunny", humidity="40%"), + "answer_any": ["31"], + }, + { # 9: diacritics must survive argument extraction + "id": "S9", "name": "argument fidelity", + "tools": [weather_tool()], + "messages": [{"role": "user", "content": "What's the current weather in České Budějovice? Use the tool."}], + "expect_tool": "get_weather", "expect_args": {"city": "budejovice"}, + "phase2": True, + "result": tool_result("Ceske Budejovice", temperature_c=23, condition="cloudy", wind_kmh=8), + "answer_any": ["23"], + }, + { # 10 hardest: non-obvious tool among 5, arg is a ticker not a city + "id": "S10", "name": "distractor selection", + "tools": [weather_tool(), get_time_tool(), get_stock_tool(), send_email_tool(), translate_tool()], + "messages": [{"role": "user", "content": "How are Google shares doing today?"}], + "expect_tool": "get_stock_price", "expect_args": {"symbol": "goog"}, + "phase2": True, + "result": json.dumps({"symbol": "GOOG", "price_usd": 172.5, "currency": "USD", "change_pct": 1.2}), + "answer_any": ["172", "goog"], + }, +] + + +def parse_args(tc): + """Return (name, args_dict) from a tool_call entry, tolerating str/dict args.""" + fn = tc.get("function", {}) + name = fn.get("name", "") + raw = fn.get("arguments", {}) + if raw is None: + raw = {} + if isinstance(raw, str): + try: + raw = json.loads(raw) + except json.JSONDecodeError: + raw = None + return name, (raw if isinstance(raw, dict) else None) + + +def args_ok(args, expected): + if not isinstance(args, dict): + return False, "arguments not a valid object" + for key, substr in expected.items(): + val = str(args.get(key, "")) + if substr.lower() not in val.lower() or (substr.lower() not in strip_accents(val.lower()) and False): + if substr.lower() not in strip_accents(val.lower()): + return False, f"{key}={args.get(key)!r} (want substring {substr!r})" + return True, "" + + +def answer_ok(answer, needles): + low = strip_accents(answer.lower()) + return any(n.lower() in low for n in needles) + + +def run_scenario(model, sc): + global LAST_TIMING + msgs = list(sc["messages"]) + try: + r1 = chat({"model": model, "messages": msgs, "tools": sc["tools"], "stream": False}) + except Exception as e: + print(f"PHASE1 ERROR: {e}") + return "ERROR" + LAST_TIMING = { + "load_ms": r1.get("load_duration") or 0, + "prompt_ms": r1.get("prompt_eval_duration") or 0, + "eval_ms": r1.get("eval_duration") or 0, + } + m1 = r1.get("message", {}) + calls = m1.get("tool_calls") or [] + + # --- no-call scenario --- + if sc.get("no_call"): + if calls: + print(f" called {calls[0].get('function', {}).get('name', '?')} despite no tool needed -> FAIL") + return "FAIL" + content = m1.get("content", "") or "" + ok = answer_ok(content, sc["answer_any"]) + print(f" no tool call: ok; answer: {trunc(content, 150)}") + return "PASS" if ok else "FAIL" + + # --- multi-call scenario --- + if sc.get("multi_call"): + found = set() + names_ok = True + for tc in calls: + name, args = parse_args(tc) + if name != "get_weather": + names_ok = False + if isinstance(args, dict): + found.add(strip_accents(str(args.get("city", "")).lower())) + want = set(sc["multi_call"]) + hit = want & found + verdict = "FAIL" if not calls else ("PASS" if hit == want else "PARTIAL") + print(f" calls: {len(calls)}; cities: {sorted(found) or 'none'}; names_ok={names_ok}") + return verdict + + # --- standard call scenario --- + if not calls: + print(f" no tool_calls; content: {trunc(m1.get('content', ''), 180)}") + return "FAIL" + name, args = parse_args(calls[0]) + print(f" call: {name} {json.dumps(args, ensure_ascii=False) if args else calls[0]}") + p1 = "PASS" + if name != sc["expect_tool"]: + p1 = "PARTIAL" + else: + ok, why = args_ok(args, sc.get("expect_args", {})) + if not ok: + p1 = "PARTIAL" + print(f" arg mismatch: {why}") + print(f" PHASE1 -> {p1}") + + if not sc.get("phase2"): + return p1 + + # --- phase 2: feed result back --- + msgs.append(m1) + msgs.append({"role": "tool", "name": sc["expect_tool"], "content": sc["result"]}) + try: + r2 = chat({"model": model, "messages": msgs, "tools": sc["tools"], "stream": False}) + except Exception as e: + print(f"PHASE2 ERROR: {e}") + return "ERROR" + answer = (r2.get("message", {}) or {}).get("content", "") or "" + p2ok = answer_ok(answer, sc["answer_any"]) + print(f" answer: {trunc(answer, 200)}") + print(f" PHASE2 -> {'PASS' if p2ok else 'FAIL'}") + + if p1 == "PASS": + return "PASS" if p2ok else "PARTIAL" + return p1 if p1 == "FAIL" else "PARTIAL" + + +def main(): + if len(sys.argv) < 2: + print(f"usage: {sys.argv[0]} \nmodels (smallest first): {', '.join(MODELS)}") + sys.exit(2) + model = sys.argv[1] + if model not in MODELS: + print(f"unknown model {model!r}; known: {', '.join(MODELS)}") + sys.exit(2) + + print(f"MODEL: {model} — {len(SCENARIOS)} scenarios, English prompts\n" + "=" * 70) + score, verdicts, t0 = 0.0, {}, time.monotonic() + aborted = False + for sc in SCENARIOS: + print(f"\n{sc['id']} {sc['name']}") + try: + v = run_scenario(model, sc) + except KeyboardInterrupt: + v = "ERROR" + if v == "ERROR": + print(" -> aborting remaining scenarios for this model (no response in reasonable time)") + aborted = True + verdicts[sc["id"]] = v + score += {"PASS": 1.0, "PARTIAL": 0.5}.get(v, 0.0) + if v != "ERROR": + print(f" => {v}") + if aborted: + break + + dt = time.monotonic() - t0 + print(f"\n{'=' * 70}\nSUMMARY {model}") + line = ", ".join(f"{sid}:{v}" for sid, v in verdicts.items()) + print(line) + n = sum(1 for v in verdicts.values() if v != "ERROR") + print(f"SCORE: {score:.1f}/{len(SCENARIOS)} ({n}/{len(SCENARIOS)} scenarios answered)") + t = LAST_TIMING + if t: + load = t.get("load_ms", 0) / 1e9 + print(f"timing of last request: load {load:.1f}s, prompt {t.get('prompt_ms', 0)/1e6:.0f}ms, eval {t.get('eval_ms', 0)/1e6:.0f}ms (load only on first run of a model)") + + +if __name__ == "__main__": + main() diff --git a/projects/ai/memory.md b/projects/ai/memory.md index 4563f08..c72477a 100644 --- a/projects/ai/memory.md +++ b/projects/ai/memory.md @@ -63,3 +63,27 @@ https://www.librechat.ai/docs/local/docker **Zjištěné výsledky (2026-09-12):** 57 modelů na instanci; lokálních s `tools` pod 10 GB cca 24 (nejmenší `lfm2.5-thinking` 0.7 GB, největší `gemma4`/`gemma4:e4b` 9.6 GB). `:cloud` modely mají size 0 GB (pointer na Ollama Cloud, neběží lokálně — do kritéria velikosti na disku nepatří). `gemini-3-flash-preview:cloud` vrací `POST /api/show` HTTP 410 Gone — mrtvý pointer, Ollama Cloud model stáhla. **Otevřené:** funkční test tool callingu (reálný `tools` request na každý kandidát a ověření validního `tool_calls` v odpovědi) ještě neproveden — nabídka visí. +- 2026-09-13: Funkční test tool callingu lokálních Ollama modelů na `nvidia.hell` dokončen (18 modelů, 10 scénářů, anglické prompty). + +**Skript:** `projects/ai/artifacts/ollama-toolcall-test.py` — 10 scénářů rostoucí obtížnosti (S1 direct call, S2 implied call, S3 tool selection ze 3, S4 context resolution, S5 více parametrů, S6 no-tool restraint — NESMÍ volat, S7 multi-call dvě volání, S8 result reasoning, S9 diakritika v argumentu, S10 distractor selection z 5 nástrojů). Každý scénář 2 fáze: vyslat správný tool call + zpracovat vrácený výsledek. Spouští se per model: `PYTHONUNBUFFERED=1 uv run ollama-toolcall-test.py `. Přísný parser (odmítl vnořené arguments u nemotronu — reálný klient unwrap nedělá). + +**Čisté 10/10:** granite4.1:8b (5.3 GB, 26s, nejlepší balance), lfm2.5-thinking (0.7 GB, 27s — pozor, 1B model, slabá sémantika, ale mechanika solidní), ministral-3 (6.0 GB, 27.5s, nejrychlejší eval ~332ms), ornith (5.6 GB, 46s), qwen3.5:9b (6.6 GB, 73s), qwen3 (5.2 GB, 89s — pomalý, thinking model). + +**Selhání podle vzorců:** +- Over-eager calling (S6 fail — volá i když nemá): deepseek-v3, llama3.2 (jinak 9/10). +- S7 multi-call bug (jen jedno volání z dvou): granite4 (9.5/10), granite4:tiny-h (8/10). +- Prázdné odpovědi/args: granite4:tiny-h (S3), granite3.3 (S7.5/10 — S6 odpověděl prázdný text, S3 prázdné arguments). +- Iniciativa (umí volat jen na explicitní pokyn, jinak čeká/odpovídá textem): mistral-nemo (7/10 — S4/S8 „Shall I proceed?", S8 vrátil giphy GIF link), mistral (5/10 — při implied call vypíše JSON volání jako text a **halucinuje výsledek**, S6 odpověděl voláním get_weather s city=Berlin). +- Template mismatch (tool call jako plaintext, syrové speciální tokeny): phi4-mini (1.5/10), phi4-mini-reasoning (netestovatelný — reasoning CoT tolik zpomalí, že ~3 min bez odpovědi). +- Vnořené arguments (`{"type":..., "arguments":{...}}` místo přímých parametrů — sémanticky správně, formálně rozbité pro standardního klienta): nemotron-mini (5.5/10), dřív i aya-expanse. +- Sejmuly server (memory watchdog na nvidia.hell zabil llama-server — `Remote end closed connection`): aya-expanse (5.1 GB, hned na S1), qwen3.5 (6.6 GB, na S10 s 5 nástroji; do S9 byl 9/9 PASS, qwen3.5:9b S10 ale zvládl — rozdíl mezi tagy nebo náhoda v načítání). +- Vynechané dle zadání uživatele: gemma4-uncensored, qwen3.5-uncensored; gemma4/gemma4:e4b (9.6 GB) dřív sejmul server, scope ukončen u mistral-nemo (7.1 GB). + +**Poznatky pro budoucí testy:** +1. Jeden prompt nestačí — s českým promptem lfm2.5-thinking failoval na S1 (city=„Berlin"), s anglickým 10/10. Malé modely rozhoduje jazyk promptu, ne schopnost. +2. S6 no-tool restraint je filtr, který vypadlo víc modelů než kterákoli jiná kategorie. +3. Více nástrojů v promptu = vyšší paměťové nároky = watchdog kill. Testovat vícenástrojové scénáře jen u finálních kandidátů. +4. Rodinné vzorce: granite = disciplína (S6 umí), qwen = spolehlivý tool calling, phi4 = rozbitý, mistral rodina = problém s iniciativou. +5. Čas v summary skriptu je wall-clock celého běhu včetně prvního načtení modelu; `load_duration` z API se vypisuje zvlášť (load>0 jen při prvním requestu). + +**Doporučení pro agenta / tool use:** granite4.1:8b (nejlepší balance), ministral-3 (nejrychlejší), qwen3 (spolehlivost, pomalejší).