#!/usr/bin/env python3 """Functional tool-calling test suite for local Ollama models (<10 GB, tools capability). 10 scenarios per model, increasing difficulty, English-only prompts. S1 direct call explicit "use the tool", simple city arg S2 implied call no explicit tool instruction, model must infer S3 tool selection 3 tools available, must pick get_weather S4 context resolution city referenced indirectly ("there") from context S5 multiple parameters city + unit, unit must be extracted from request S6 no-tool restraint answerable without tools -> must NOT call S7 multi-call two cities compared -> both calls expected S8 result reasoning use result to answer a yes/no derived question S9 argument fidelity city with diacritics must be preserved S10 distractor selection 5 tools, must pick the non-obvious get_stock_price Per scenario: PHASE 1 (emit valid tool call) + PHASE 2 (use the returned result). Verdicts: PASS = 1, PARTIAL = 0.5, FAIL = 0. Prints each scenario result immediately (unbuffered). Usage: PYTHONUNBUFFERED=1 uv run ollama-toolcall-test.py """ import json import sys import time import unicodedata import urllib.request BASE = "http://nvidia.hell:11434" TIMEOUT = 120 # per request; abort the model run if exceeded MODELS = [ "lfm2.5-thinking", # 0.7 "deepseek-v3", # 2.0 "llama3.2", # 2.0 "granite4", # 2.1 "phi4-mini", # 2.5 "nemotron-mini", # 2.7 "phi4-mini-reasoning", # 3.2 "mistral", # 4.1 "granite4:tiny-h", # 4.2 "granite3.3", # 4.9 "aya-expanse", # 5.1 "qwen3", # 5.2 "granite4.1:8b", # 5.3 "gemma4-uncensored", # 5.3 "ornith", # 5.6 "qwen3.5-uncensored", # 5.6 "ministral-3", # 6.0 "qwen3.5:9b", # 6.6 "qwen3.5", # 6.6 "mistral-nemo", # 7.1 ] def strip_accents(s): return "".join(c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn") def trunc(s, n=200): s = (s or "").replace("\n", " ").strip() return s[:n] + ("…" if len(s) > n else "") LAST_TIMING = {} def chat(payload, timeout=TIMEOUT): req = urllib.request.Request( f"{BASE}/api/chat", data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}, ) with urllib.request.urlopen(req, timeout=timeout) as r: return json.load(r) def fdef(name, description, properties, required): return {"type": "function", "function": { "name": name, "description": description, "parameters": {"type": "object", "properties": properties, "required": required}, }} W_CITY = {"city": {"type": "string", "description": "City name"}} def weather_tool(unit=False): props = dict(W_CITY) if unit: props["unit"] = {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit for the result"} return fdef("get_weather", "Get the current weather for a city", props, ["city"]) def get_time_tool(): return fdef("get_time", "Get the current local time in a timezone", {"timezone": {"type": "string", "description": "IANA timezone, e.g. Europe/Prague"}}, ["timezone"]) def get_currency_tool(): return fdef("get_currency_rate", "Get the current exchange rate between two currencies", {"from": {"type": "string"}, "to": {"type": "string"}}, ["from", "to"]) def get_stock_tool(): return fdef("get_stock_price", "Get the current stock price for a company", {"symbol": {"type": "string", "description": "Stock ticker symbol, e.g. GOOG"}}, ["symbol"]) def send_email_tool(): return fdef("send_email", "Send an email to someone", {"to": {"type": "string", "description": "Recipient email"}, "subject": {"type": "string"}, "body": {"type": "string"}}, ["to"]) def translate_tool(): return fdef("translate_text", "Translate a text to another language", {"text": {"type": "string"}, "target_language": {"type": "string"}}, ["text"]) def tool_result(city, **kw): d = {"city": city} d.update(kw) return json.dumps(d) SCENARIOS = [ { # 1 easy: explicit instruction, trivial extraction "id": "S1", "name": "direct call", "tools": [weather_tool()], "messages": [{"role": "user", "content": "What is the current weather in Brno right now? Use the tool."}], "expect_tool": "get_weather", "expect_args": {"city": "brno"}, "phase2": True, "result": tool_result("Brno", temperature_c=18, condition="partly cloudy", wind_kmh=7), "answer_any": ["18"], }, { # 2: no "use the tool" hint at all "id": "S2", "name": "implied call", "tools": [weather_tool()], "messages": [{"role": "user", "content": "I'm about to head out. Can you check what the weather's like in Prague?"}], "expect_tool": "get_weather", "expect_args": {"city": "prague"}, "phase2": True, "result": tool_result("Prague", temperature_c=14, condition="overcast", wind_kmh=10), "answer_any": ["14"], }, { # 3: pick the right tool out of 3 "id": "S3", "name": "tool selection", "tools": [weather_tool(), get_time_tool(), get_currency_tool()], "messages": [{"role": "user", "content": "Should I take an umbrella when going to work in Ostrava today?"}], "expect_tool": "get_weather", "expect_args": {"city": "ostrava"}, "phase2": True, "result": tool_result("Ostrava", temperature_c=12, condition="rain", wind_kmh=20, precipitation_prob="85%"), "answer_any": ["umbrella", "rain"], }, { # 4: city only implied by conversation context "id": "S4", "name": "context resolution", "tools": [weather_tool()], "messages": [ {"role": "user", "content": "My sister lives in Pardubice."}, {"role": "assistant", "content": "Nice! Pardubice is a lovely city. Is there anything I can help you with?"}, {"role": "user", "content": "What's the weather like there right now?"}, ], "expect_tool": "get_weather", "expect_args": {"city": "pardubice"}, "phase2": True, "result": tool_result("Pardubice", temperature_c=21, condition="clear", wind_kmh=5), "answer_any": ["21"], }, { # 5: two parameters, second one from wording "id": "S5", "name": "multiple parameters", "tools": [weather_tool(unit=True)], "messages": [{"role": "user", "content": "What's the weather in Vienna? I'd like it in Fahrenheit."}], "expect_tool": "get_weather", "expect_args": {"city": "vienna", "unit": "fahr"}, "phase2": True, "result": json.dumps({"city": "Vienna", "temperature_f": 64, "condition": "sunny", "humidity": "35%"}), "answer_any": ["64"], }, { # 6: must NOT call any tool "id": "S6", "name": "no-tool restraint", "tools": [weather_tool()], "messages": [{"role": "user", "content": "What is the capital of Germany?"}], "no_call": True, "phase2": False, "answer_any": ["berlin"], }, { # 7: two tool calls in one answer "id": "S7", "name": "multi-call", "tools": [weather_tool()], "messages": [{"role": "user", "content": "Compare the current weather in Brno and Olomouc."}], "multi_call": ["brno", "olomouc"], "phase2": False, }, { # 8: answer a derived yes/no question from the result "id": "S8", "name": "result reasoning", "tools": [weather_tool()], "messages": [{"role": "user", "content": "Is it too hot for a run in Plzen right now?"}], "expect_tool": "get_weather", "expect_args": {"city": "plzen"}, "phase2": True, "result": tool_result("Plzen", temperature_c=31, condition="sunny", humidity="40%"), "answer_any": ["31"], }, { # 9: diacritics must survive argument extraction "id": "S9", "name": "argument fidelity", "tools": [weather_tool()], "messages": [{"role": "user", "content": "What's the current weather in České Budějovice? Use the tool."}], "expect_tool": "get_weather", "expect_args": {"city": "budejovice"}, "phase2": True, "result": tool_result("Ceske Budejovice", temperature_c=23, condition="cloudy", wind_kmh=8), "answer_any": ["23"], }, { # 10 hardest: non-obvious tool among 5, arg is a ticker not a city "id": "S10", "name": "distractor selection", "tools": [weather_tool(), get_time_tool(), get_stock_tool(), send_email_tool(), translate_tool()], "messages": [{"role": "user", "content": "How are Google shares doing today?"}], "expect_tool": "get_stock_price", "expect_args": {"symbol": "goog"}, "phase2": True, "result": json.dumps({"symbol": "GOOG", "price_usd": 172.5, "currency": "USD", "change_pct": 1.2}), "answer_any": ["172", "goog"], }, ] def parse_args(tc): """Return (name, args_dict) from a tool_call entry, tolerating str/dict args.""" fn = tc.get("function", {}) name = fn.get("name", "") raw = fn.get("arguments", {}) if raw is None: raw = {} if isinstance(raw, str): try: raw = json.loads(raw) except json.JSONDecodeError: raw = None return name, (raw if isinstance(raw, dict) else None) def args_ok(args, expected): if not isinstance(args, dict): return False, "arguments not a valid object" for key, substr in expected.items(): val = str(args.get(key, "")) if substr.lower() not in val.lower() or (substr.lower() not in strip_accents(val.lower()) and False): if substr.lower() not in strip_accents(val.lower()): return False, f"{key}={args.get(key)!r} (want substring {substr!r})" return True, "" def answer_ok(answer, needles): low = strip_accents(answer.lower()) return any(n.lower() in low for n in needles) def run_scenario(model, sc): global LAST_TIMING msgs = list(sc["messages"]) try: r1 = chat({"model": model, "messages": msgs, "tools": sc["tools"], "stream": False}) except Exception as e: print(f"PHASE1 ERROR: {e}") return "ERROR" LAST_TIMING = { "load_ms": r1.get("load_duration") or 0, "prompt_ms": r1.get("prompt_eval_duration") or 0, "eval_ms": r1.get("eval_duration") or 0, } m1 = r1.get("message", {}) calls = m1.get("tool_calls") or [] # --- no-call scenario --- if sc.get("no_call"): if calls: print(f" called {calls[0].get('function', {}).get('name', '?')} despite no tool needed -> FAIL") return "FAIL" content = m1.get("content", "") or "" ok = answer_ok(content, sc["answer_any"]) print(f" no tool call: ok; answer: {trunc(content, 150)}") return "PASS" if ok else "FAIL" # --- multi-call scenario --- if sc.get("multi_call"): found = set() names_ok = True for tc in calls: name, args = parse_args(tc) if name != "get_weather": names_ok = False if isinstance(args, dict): found.add(strip_accents(str(args.get("city", "")).lower())) want = set(sc["multi_call"]) hit = want & found verdict = "FAIL" if not calls else ("PASS" if hit == want else "PARTIAL") print(f" calls: {len(calls)}; cities: {sorted(found) or 'none'}; names_ok={names_ok}") return verdict # --- standard call scenario --- if not calls: print(f" no tool_calls; content: {trunc(m1.get('content', ''), 180)}") return "FAIL" name, args = parse_args(calls[0]) print(f" call: {name} {json.dumps(args, ensure_ascii=False) if args else calls[0]}") p1 = "PASS" if name != sc["expect_tool"]: p1 = "PARTIAL" else: ok, why = args_ok(args, sc.get("expect_args", {})) if not ok: p1 = "PARTIAL" print(f" arg mismatch: {why}") print(f" PHASE1 -> {p1}") if not sc.get("phase2"): return p1 # --- phase 2: feed result back --- msgs.append(m1) msgs.append({"role": "tool", "name": sc["expect_tool"], "content": sc["result"]}) try: r2 = chat({"model": model, "messages": msgs, "tools": sc["tools"], "stream": False}) except Exception as e: print(f"PHASE2 ERROR: {e}") return "ERROR" answer = (r2.get("message", {}) or {}).get("content", "") or "" p2ok = answer_ok(answer, sc["answer_any"]) print(f" answer: {trunc(answer, 200)}") print(f" PHASE2 -> {'PASS' if p2ok else 'FAIL'}") if p1 == "PASS": return "PASS" if p2ok else "PARTIAL" return p1 if p1 == "FAIL" else "PARTIAL" def main(): if len(sys.argv) < 2: print(f"usage: {sys.argv[0]} \nmodels (smallest first): {', '.join(MODELS)}") sys.exit(2) model = sys.argv[1] if model not in MODELS: print(f"unknown model {model!r}; known: {', '.join(MODELS)}") sys.exit(2) print(f"MODEL: {model} — {len(SCENARIOS)} scenarios, English prompts\n" + "=" * 70) score, verdicts, t0 = 0.0, {}, time.monotonic() aborted = False for sc in SCENARIOS: print(f"\n{sc['id']} {sc['name']}") try: v = run_scenario(model, sc) except KeyboardInterrupt: v = "ERROR" if v == "ERROR": print(" -> aborting remaining scenarios for this model (no response in reasonable time)") aborted = True verdicts[sc["id"]] = v score += {"PASS": 1.0, "PARTIAL": 0.5}.get(v, 0.0) if v != "ERROR": print(f" => {v}") if aborted: break dt = time.monotonic() - t0 print(f"\n{'=' * 70}\nSUMMARY {model}") line = ", ".join(f"{sid}:{v}" for sid, v in verdicts.items()) print(line) n = sum(1 for v in verdicts.values() if v != "ERROR") print(f"SCORE: {score:.1f}/{len(SCENARIOS)} ({n}/{len(SCENARIOS)} scenarios answered)") t = LAST_TIMING if t: load = t.get("load_ms", 0) / 1e9 print(f"timing of last request: load {load:.1f}s, prompt {t.get('prompt_ms', 0)/1e6:.0f}ms, eval {t.get('eval_ms', 0)/1e6:.0f}ms (load only on first run of a model)") if __name__ == "__main__": main()