upravy projektu a skillu
This commit is contained in:
lachtan
897
skills/reflect/scripts/reflect_auto.py
Normal file
897
skills/reflect/scripts/reflect_auto.py
Normal file
@@ -0,0 +1,897 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = ["nanobot-ai"]
|
||||
# ///
|
||||
"""reflect_auto.py — unattended analysis run for the /reflect skill.
|
||||
|
||||
Distils new sessions, asks the agent to diagnose recurring mistakes, and files the
|
||||
findings. **It never applies anything** — there is no edit path in this script at all.
|
||||
Applying a fix happens only in the interactive `/reflect` review, one finding at a time,
|
||||
after the user approves it.
|
||||
|
||||
Two guards protect that boundary:
|
||||
* the agent is told not to write, and a git fingerprint of the workspace is compared
|
||||
before and after the turn — if the agent wrote anything, nothing is filed;
|
||||
* a finding seen only once is filed as `watch` and stays silent. Telegram is notified
|
||||
only once a pattern repeats, so a daily run does not mean a daily notification.
|
||||
|
||||
Same external-script pattern as skills/compact-memory/scripts/compact_memory_auto.py:
|
||||
fresh never-reused session_key, delivery straight to the Telegram Bot API, and the
|
||||
message composed here rather than by the model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from reflect_distill import DEFAULT_BUDGET_CHARS, SessionDigest, collect_sessions, iter_batches
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from nanobot import Nanobot # ty: ignore[unresolved-import]
|
||||
|
||||
# A digest prompt is tens of thousands of tokens, and prefilling it on glm-5.3:cloud overruns
|
||||
# nanobot's 120s per-request default — whose retry then throws the finished prefill away and
|
||||
# starts over. Set before nanobot is imported (`_open_bot` defers it); overridable from the shell.
|
||||
os.environ.setdefault("NANOBOT_OPENAI_COMPAT_TIMEOUT_S", "600")
|
||||
os.environ.setdefault("NANOBOT_LLM_TIMEOUT_S", "900")
|
||||
|
||||
CONFIG = Path.home() / ".nanobot" / "config.json"
|
||||
WORKSPACE_FALLBACK = Path.home() / ".nanobot" / "workspace"
|
||||
|
||||
STATE_REL = "reflect/state.json"
|
||||
FINDINGS_REL = "reflect/findings.jsonl"
|
||||
RESULTS_REL = "results"
|
||||
|
||||
FALLBACK_CHAT_ID = "8826147089"
|
||||
MODEL_PRESET = "glm53"
|
||||
# The soft deadline is what actually bounds a run: no new batch starts past it, and everything
|
||||
# already analysed is on disk. TIMEOUT_SECONDS only catches a single batch that hangs.
|
||||
DEFAULT_DEADLINE_MINUTES = 20
|
||||
# The cursor is the floor of a run, this is the ceiling: findings are meant to describe recent
|
||||
# behaviour, and a months-deep backlog walked one batch a night never gets there (2026-09-02:
|
||||
# the cursor sat on 05-29 while 3 runs of findings were presented as current).
|
||||
DEFAULT_WINDOW_DAYS = 21
|
||||
TIMEOUT_SECONDS = 45 * 60
|
||||
MAX_ATTEMPTS = 3
|
||||
MAX_LLM_ERROR_RETRIES = 2
|
||||
|
||||
SEVERITIES = ("low", "medium", "high")
|
||||
STATUS_WATCH = "watch"
|
||||
STATUS_OPEN = "open"
|
||||
STATUS_APPLIED = "applied"
|
||||
STATUS_REJECTED = "rejected"
|
||||
PATCH_KEYS = frozenset({"file", "old_text", "new_text"})
|
||||
FINDING_KEYS = frozenset(
|
||||
{"pattern", "severity", "diagnosis", "evidence", "occurrences", "sessions_affected", "proposal", "patch"}
|
||||
)
|
||||
MAX_DIAGNOSIS_CHARS = 600
|
||||
MAX_PROPOSAL_CHARS = 400
|
||||
MAX_FINDINGS_PER_BATCH = 12
|
||||
# Folded findings keep evidence from earlier slices too, or a cumulative count is unauditable.
|
||||
MAX_EVIDENCE = 6
|
||||
PATTERN_RE = re.compile(r"^[a-z][a-z0-9-]{2,48}$")
|
||||
# `when` is a free string the model fills in; the shapes seen so far are `2026-08-31` and
|
||||
# `2026-08-31 13:53`, plus the occasional non-date. Only the leading date is usable.
|
||||
EVIDENCE_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
||||
JSON_FENCE = re.compile(r"```(?:json)?\s*\n(.*?)```", re.DOTALL)
|
||||
|
||||
GOAL = """\
|
||||
You are auditing your own past behaviour. Below is a distilled log of {count} of your earlier
|
||||
sessions: user and assistant messages in full, tool calls collapsed to `name(args) -> ok|ERROR`.
|
||||
|
||||
Find **recurring mistakes worth fixing** — for example a tool retried with identical arguments
|
||||
after an error instead of being diagnosed, a runaway loop, a tool chosen where a better one
|
||||
existed, a multi-step request answered without doing the work, or an instruction you clearly
|
||||
missed. Judge severity by wasted turns and by the harm to the user's result.
|
||||
|
||||
Rules:
|
||||
- Report a repeated pattern, with concrete evidence: which sessions, which turns. A single
|
||||
occurrence is worth reporting only when it matches a pattern in "Known patterns" below — reuse
|
||||
its exact id and give the honest count for this slice (1 is fine), because those counts add up
|
||||
across slices. A novel slip you have seen only once here, skip.
|
||||
- These {count} sessions are one slice of a longer history, not all of it. Count only what you can
|
||||
see in this slice; the store sums the counts across slices for you.
|
||||
- Do NOT report a pattern already listed in "Known patterns" below unless you see new occurrences;
|
||||
reuse its exact `pattern` identifier when you do.
|
||||
- `pattern` is a short stable kebab-case id (e.g. `retry-without-diagnosis`). Reuse existing ids
|
||||
from the list rather than inventing a new name for the same thing.
|
||||
- Include a `patch` only when you are confident: `old_text` must be copied character-for-character
|
||||
from the current file and must occur exactly once in it. When unsure, give `proposal` alone.
|
||||
- **Read at most 2 files**, and only to copy `old_text` for a patch. Every extra read costs another
|
||||
full pass over this whole digest, so spend those two reads on a patch you are sure about.
|
||||
- At most {max_findings} findings. Fewer, well-evidenced findings are better than a long list.
|
||||
- **Use no quotation marks of any kind inside JSON string values.** Quote a log line by
|
||||
writing it plainly, without wrapping it in quotes. A stray `"` breaks the whole answer.
|
||||
|
||||
**Write nothing.** Do not edit, create or delete any file, and do not run any command that
|
||||
changes state. This is an analysis-only run; a human reviews and applies your suggestions later.
|
||||
|
||||
Answer with exactly one ```json code block and nothing else — no narration before or after:
|
||||
|
||||
```json
|
||||
{{"findings": [{{"pattern": "retry-without-diagnosis", "severity": "medium",
|
||||
"diagnosis": "After an HTTP error the same call is repeated with identical arguments…",
|
||||
"evidence": [{{"session": "websocket_e5a6aa…", "when": "2026-07-11", "excerpt": "web_fetch → ERROR 403 ×4"}}],
|
||||
"occurrences": 7, "sessions_affected": 4,
|
||||
"proposal": "Add a hard STOP gate to the Fetching section",
|
||||
"patch": {{"file": "skills/flight-search/SKILL.md", "old_text": "…", "new_text": "…"}}}}]}}
|
||||
```
|
||||
|
||||
If you find nothing worth reporting, answer with an empty `findings` list.
|
||||
|
||||
## Known patterns
|
||||
|
||||
{known_patterns}
|
||||
|
||||
## Sessions
|
||||
|
||||
{digest}
|
||||
"""
|
||||
|
||||
RETRY_PROMPT = """\
|
||||
Your findings were rejected by the validator:
|
||||
|
||||
{errors}
|
||||
|
||||
Answer again with exactly one corrected ```json code block and nothing else.
|
||||
"""
|
||||
|
||||
|
||||
class ReflectError(Exception):
|
||||
"""A failure worth reporting to the user in one line."""
|
||||
|
||||
|
||||
class FindingsError(ReflectError):
|
||||
"""The agent's answer is malformed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Finding:
|
||||
"""One diagnosed pattern, as filed in reflect/findings.jsonl."""
|
||||
|
||||
id: str
|
||||
status: str
|
||||
created: str
|
||||
last_seen: str
|
||||
pattern: str
|
||||
severity: str
|
||||
diagnosis: str
|
||||
evidence: tuple[dict[str, str], ...]
|
||||
occurrences: int
|
||||
sessions_affected: int
|
||||
proposal: str
|
||||
patch: dict[str, str] | None = None
|
||||
regression_of: str | None = None
|
||||
history: tuple[str, ...] = field(default_factory=tuple)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
record = {
|
||||
"id": self.id,
|
||||
"status": self.status,
|
||||
"created": self.created,
|
||||
"last_seen": self.last_seen,
|
||||
"pattern": self.pattern,
|
||||
"severity": self.severity,
|
||||
"diagnosis": self.diagnosis,
|
||||
"evidence": list(self.evidence),
|
||||
"occurrences": self.occurrences,
|
||||
"sessions_affected": self.sessions_affected,
|
||||
"proposal": self.proposal,
|
||||
}
|
||||
if self.patch:
|
||||
record["patch"] = self.patch
|
||||
if self.regression_of:
|
||||
record["regression_of"] = self.regression_of
|
||||
if self.history:
|
||||
record["history"] = list(self.history)
|
||||
return record
|
||||
|
||||
|
||||
def _config() -> dict[str, Any]:
|
||||
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _workspace(config: dict[str, Any]) -> Path:
|
||||
configured = config.get("agents", {}).get("defaults", {}).get("workspace")
|
||||
return Path(configured).expanduser() if configured else WORKSPACE_FALLBACK
|
||||
|
||||
|
||||
def _telegram_config(config: dict[str, Any]) -> tuple[str, str]:
|
||||
telegram = config["channels"]["telegram"]
|
||||
allow_from = telegram.get("allowFrom") or []
|
||||
chat_id = str(allow_from[0]) if allow_from else FALLBACK_CHAT_ID
|
||||
return telegram["token"], chat_id
|
||||
|
||||
|
||||
def _send_telegram(text: str, token: str, chat_id: str) -> None:
|
||||
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||||
payload = urllib.parse.urlencode({"chat_id": chat_id, "text": text}).encode()
|
||||
request = urllib.request.Request(url, data=payload, method="POST")
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
response.read()
|
||||
|
||||
|
||||
def _git_fingerprint(workspace: Path) -> str:
|
||||
"""HEAD plus the porcelain status — an otherwise unchanged workspace hashes identically.
|
||||
|
||||
This is the guard that the analysis turn stayed read-only. It covers the whole tree,
|
||||
not just the files the agent was asked about.
|
||||
"""
|
||||
try:
|
||||
head = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=workspace, capture_output=True, text=True, check=True
|
||||
).stdout
|
||||
status = subprocess.run(
|
||||
["git", "status", "--porcelain"], cwd=workspace, capture_output=True, text=True, check=True
|
||||
).stdout
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as error:
|
||||
raise ReflectError(f"workspace git is unavailable, refusing to run unguarded: {error}") from error
|
||||
return head + status
|
||||
|
||||
|
||||
def _load_state(workspace: Path) -> dict[str, Any]:
|
||||
path = workspace / STATE_REL
|
||||
if not path.exists():
|
||||
return {"cursor": "", "runs": []}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _save_state(workspace: Path, state: dict[str, Any]) -> None:
|
||||
path = workspace / STATE_REL
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _load_findings(workspace: Path) -> list[dict[str, Any]]:
|
||||
path = workspace / FINDINGS_REL
|
||||
if not path.exists():
|
||||
return []
|
||||
records = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
records.append(json.loads(line))
|
||||
return records
|
||||
|
||||
|
||||
def _write_findings(workspace: Path, records: list[dict[str, Any]]) -> None:
|
||||
path = workspace / FINDINGS_REL
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
body = "\n".join(json.dumps(record, ensure_ascii=False) for record in records)
|
||||
path.write_text(body + "\n" if body else "", encoding="utf-8")
|
||||
|
||||
|
||||
def _known_patterns(records: list[dict[str, Any]]) -> str:
|
||||
"""Render the pattern vocabulary handed to the model, so it reuses ids instead of renaming."""
|
||||
if not records:
|
||||
return "_(none yet)_"
|
||||
seen: dict[str, dict[str, Any]] = {}
|
||||
for record in records:
|
||||
current = seen.get(record["pattern"])
|
||||
if not current or record.get("occurrences", 0) > current.get("occurrences", 0):
|
||||
seen[record["pattern"]] = record
|
||||
rows = []
|
||||
for pattern, record in sorted(seen.items()):
|
||||
rows.append(f"- `{pattern}` [{record['status']}] — {record['diagnosis'][:120]}")
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _new_id(existing: set[str]) -> str:
|
||||
while True:
|
||||
candidate = f"f{uuid.uuid4().hex[:4]}"
|
||||
if candidate not in existing:
|
||||
return candidate
|
||||
|
||||
|
||||
def _text_field(raw: Any, name: str, index: int, limit: int) -> str:
|
||||
"""Trimmed non-empty string, cut to `limit` — an overlong answer is shortened, not discarded."""
|
||||
if not isinstance(raw, str) or not raw.strip():
|
||||
raise FindingsError(f'- finding {index}: "{name}" must be a non-empty string')
|
||||
text = raw.strip()
|
||||
return text if len(text) <= limit else text[: limit - 1] + "…"
|
||||
|
||||
|
||||
def _positive_int(raw: Any, name: str, index: int) -> int:
|
||||
if not isinstance(raw, int) or isinstance(raw, bool) or raw < 1:
|
||||
raise FindingsError(f'- finding {index}: "{name}" must be an integer >= 1')
|
||||
return raw
|
||||
|
||||
|
||||
def _parse_evidence(raw: Any, index: int) -> tuple[dict[str, str], ...]:
|
||||
if not isinstance(raw, list) or not raw:
|
||||
raise FindingsError(f'- finding {index}: "evidence" must be a non-empty list')
|
||||
items = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict) or not item.get("session"):
|
||||
raise FindingsError(f'- finding {index}: every evidence item needs a "session"')
|
||||
items.append({key: str(value) for key, value in item.items() if key in ("session", "when", "excerpt")})
|
||||
return tuple(items)
|
||||
|
||||
|
||||
def _parse_patch(raw: Any, index: int) -> dict[str, str] | None:
|
||||
"""Validate a proposed patch. Shape only — whether it still applies is checked at review time."""
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, dict):
|
||||
raise FindingsError(f'- finding {index}: "patch" must be an object or omitted')
|
||||
unknown = sorted(str(key) for key in set(raw) - PATCH_KEYS)
|
||||
if unknown:
|
||||
raise FindingsError(f"- finding {index}: patch has unknown fields {unknown}")
|
||||
missing = sorted(PATCH_KEYS - set(raw))
|
||||
if missing:
|
||||
raise FindingsError(f"- finding {index}: patch is missing {missing}")
|
||||
if not all(isinstance(raw[key], str) for key in PATCH_KEYS):
|
||||
raise FindingsError(f"- finding {index}: patch fields must be strings")
|
||||
if not raw["old_text"].strip():
|
||||
raise FindingsError(f'- finding {index}: patch "old_text" must not be empty')
|
||||
if raw["old_text"] == raw["new_text"]:
|
||||
raise FindingsError(f"- finding {index}: patch changes nothing")
|
||||
if Path(raw["file"]).is_absolute() or ".." in Path(raw["file"]).parts:
|
||||
raise FindingsError(f'- finding {index}: patch "file" must be a path inside the workspace')
|
||||
return {key: raw[key] for key in ("file", "old_text", "new_text")}
|
||||
|
||||
|
||||
def _decode_payload(content: str) -> tuple[Any, str]:
|
||||
"""Return (payload, "") or (None, an error the model can act on).
|
||||
|
||||
Naming the offending line, column and surrounding text matters: the usual failure is
|
||||
an unescaped `"` inside a Czech quotation (`„…"`) in a diagnosis, and a bare
|
||||
"no parseable json" message gives the retry nothing to fix.
|
||||
"""
|
||||
# Fenced blocks first, last one first: that is where the answer is supposed to be, so
|
||||
# both the successful parse and the reported error come from there rather than from
|
||||
# the whole reply (whose column numbers would be meaningless to the model).
|
||||
candidates = [*reversed(JSON_FENCE.findall(content)), content]
|
||||
best_error = ""
|
||||
for candidate in candidates:
|
||||
text = candidate.strip()
|
||||
if not text:
|
||||
continue
|
||||
try:
|
||||
return json.loads(text), ""
|
||||
except json.JSONDecodeError as error:
|
||||
if not best_error:
|
||||
excerpt = text[max(0, error.pos - 90) : error.pos + 30].replace("\n", " ")
|
||||
best_error = (
|
||||
f"- the JSON is invalid: {error.msg} at line {error.lineno} column {error.colno}\n"
|
||||
f"- around: …{excerpt}…\n"
|
||||
'- most likely an unescaped double quote inside a string value; write \\" '
|
||||
"or drop the quotes entirely"
|
||||
)
|
||||
return None, best_error or "- the answer contains no parseable ```json block"
|
||||
|
||||
|
||||
def _parse_finding(raw: Any, index: int, session_count: int) -> tuple[dict[str, Any], list[str]]:
|
||||
"""Validate one finding, returning it with any notes about what had to be salvaged.
|
||||
|
||||
Raises only when the record itself is unusable. A bad patch costs the patch, not the
|
||||
finding — the diagnosis and the proposal are still worth putting in front of the user.
|
||||
"""
|
||||
if not isinstance(raw, dict):
|
||||
raise FindingsError(f"- finding {index}: must be a JSON object")
|
||||
pattern = raw.get("pattern")
|
||||
if not isinstance(pattern, str) or not PATTERN_RE.match(pattern):
|
||||
raise FindingsError(f'- finding {index}: "pattern" must be a kebab-case id like `retry-without-diagnosis`')
|
||||
severity = raw.get("severity")
|
||||
if severity not in SEVERITIES:
|
||||
raise FindingsError(f'- finding {index}: "severity" must be one of {list(SEVERITIES)}')
|
||||
|
||||
notes = []
|
||||
unknown = sorted(str(key) for key in set(raw) - FINDING_KEYS)
|
||||
if unknown:
|
||||
notes.append(f"- finding {index}: ignored unknown fields {unknown}")
|
||||
try:
|
||||
patch = _parse_patch(raw.get("patch"), index)
|
||||
except FindingsError as error:
|
||||
patch = None
|
||||
notes.append(f"{error} — kept the finding without it")
|
||||
|
||||
# The counts are the model's own arithmetic and drive both the threshold and the ranking, so
|
||||
# they get the one check that needs no second opinion: a pattern cannot touch more sessions
|
||||
# than the times it occurred, nor more than the slice even held. Clamping beats re-asking —
|
||||
# a whole turn is far too expensive to spend on one wrong integer.
|
||||
occurrences = _positive_int(raw.get("occurrences"), "occurrences", index)
|
||||
sessions = _positive_int(raw.get("sessions_affected"), "sessions_affected", index)
|
||||
ceiling = min(occurrences, session_count)
|
||||
if sessions > ceiling:
|
||||
notes.append(f'- finding {index}: "sessions_affected" {sessions} exceeds {ceiling}, clamped')
|
||||
sessions = ceiling
|
||||
|
||||
finding = {
|
||||
"pattern": pattern,
|
||||
"severity": severity,
|
||||
"diagnosis": _text_field(raw.get("diagnosis"), "diagnosis", index, MAX_DIAGNOSIS_CHARS),
|
||||
"evidence": _parse_evidence(raw.get("evidence"), index),
|
||||
"occurrences": occurrences,
|
||||
"sessions_affected": sessions,
|
||||
"proposal": _text_field(raw.get("proposal"), "proposal", index, MAX_PROPOSAL_CHARS),
|
||||
"patch": patch,
|
||||
}
|
||||
return finding, notes
|
||||
|
||||
|
||||
def parse_findings(content: str, session_count: int) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
"""Parse the agent's answer into raw finding dicts, plus notes on anything dropped.
|
||||
|
||||
Only an unusable *answer* raises: a single malformed finding is discarded and the rest of
|
||||
the batch survives. Re-asking costs a whole ~420k token turn, far too much to spend on one
|
||||
bad record — the plan calls for discarding it (plans/reflect-skill.md).
|
||||
"""
|
||||
payload, decode_error = _decode_payload(content)
|
||||
if payload is None:
|
||||
raise FindingsError(decode_error)
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("findings"), list):
|
||||
raise FindingsError('- the JSON must be an object with a "findings" list')
|
||||
|
||||
raw_findings = payload["findings"]
|
||||
problems: list[str] = []
|
||||
if len(raw_findings) > MAX_FINDINGS_PER_BATCH:
|
||||
problems.append(f"- kept the first {MAX_FINDINGS_PER_BATCH} of {len(raw_findings)} findings")
|
||||
raw_findings = raw_findings[:MAX_FINDINGS_PER_BATCH]
|
||||
|
||||
parsed = []
|
||||
for index, raw in enumerate(raw_findings, start=1):
|
||||
try:
|
||||
finding, notes = _parse_finding(raw, index, session_count)
|
||||
except FindingsError as error:
|
||||
problems.append(f"{error} — finding dropped")
|
||||
continue
|
||||
parsed.append(finding)
|
||||
problems += notes
|
||||
|
||||
if raw_findings and not parsed:
|
||||
raise FindingsError("\n".join(problems))
|
||||
return parsed, problems
|
||||
|
||||
|
||||
def _fold_evidence(fresh: tuple[dict[str, str], ...], previous: Any) -> tuple[dict[str, str], ...]:
|
||||
"""Newest evidence first, older kept behind it, deduplicated and capped.
|
||||
|
||||
`occurrences` sums across runs while the evidence used to be replaced, so a folded finding
|
||||
claimed 18 occurrences and showed the two examples from the last batch — a count the user
|
||||
had no way to audit during review.
|
||||
"""
|
||||
seen: set[tuple[str, str]] = set()
|
||||
kept: list[dict[str, str]] = []
|
||||
for item in (*fresh, *(previous or ())):
|
||||
key = (item.get("session", ""), item.get("excerpt", ""))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
kept.append(item)
|
||||
if len(kept) == MAX_EVIDENCE:
|
||||
break
|
||||
return tuple(kept)
|
||||
|
||||
|
||||
def _last_seen(evidence: tuple[dict[str, str], ...], created: str) -> str:
|
||||
"""The newest date the evidence actually carries — `created` only says when it was filed."""
|
||||
dates = [item["when"][:10] for item in evidence if EVIDENCE_DATE_RE.match(item.get("when", ""))]
|
||||
return max(dates, default=created)
|
||||
|
||||
|
||||
def merge_findings(existing: list[dict[str, Any]], parsed: list[dict[str, Any]], today: str) -> list[Finding]:
|
||||
"""Apply the notification threshold and fold repeats into the record they repeat.
|
||||
|
||||
A pattern seen once stays `watch` and silent; the second sighting promotes it to `open`.
|
||||
Folding sums the counts, so it also carries the older evidence forward — a cumulative count
|
||||
with only the newest batch's examples behind it cannot be checked by the person reviewing it.
|
||||
A pattern that was already applied and comes back is a regression and opens immediately —
|
||||
but only when the evidence is newer than the fix; evidence from before it describes
|
||||
behaviour already dealt with, so the finding waits on `watch` until the window catches up.
|
||||
A pattern the user rejected never opens again — rejection is a decision, not a deferral;
|
||||
it keeps being counted and stays visible in the report and the pattern list. Rejection is
|
||||
checked against *every* record of the pattern, not just the newest one: the `watch` record
|
||||
filed after a rejection would otherwise supersede it and promote the pattern right back.
|
||||
"""
|
||||
by_pattern: dict[str, dict[str, Any]] = {}
|
||||
for record in existing:
|
||||
current = by_pattern.get(record["pattern"])
|
||||
if not current or record["created"] >= current["created"]:
|
||||
by_pattern[record["pattern"]] = record
|
||||
used_ids = {record["id"] for record in existing}
|
||||
rejected_patterns = {record["pattern"] for record in existing if record["status"] == STATUS_REJECTED}
|
||||
|
||||
results = []
|
||||
for item in parsed:
|
||||
previous = by_pattern.get(item["pattern"])
|
||||
occurrences = item["occurrences"]
|
||||
sessions = item["sessions_affected"]
|
||||
evidence = item["evidence"]
|
||||
regression_of = None
|
||||
history: tuple[str, ...] = ()
|
||||
stale_after_fix = False
|
||||
|
||||
if previous and previous["status"] in (STATUS_WATCH, STATUS_OPEN):
|
||||
occurrences += previous.get("occurrences", 0)
|
||||
sessions += previous.get("sessions_affected", 0)
|
||||
evidence = _fold_evidence(item["evidence"], previous.get("evidence"))
|
||||
history = (*previous.get("history", []), f"{previous['created']}:{previous['id']}")
|
||||
# A pattern that already came back after a fix stays a regression until it is
|
||||
# dealt with; folding the repeat in must not quietly drop the flag.
|
||||
regression_of = previous.get("regression_of")
|
||||
|
||||
last_seen = _last_seen(evidence, today)
|
||||
if previous and previous["status"] == STATUS_APPLIED:
|
||||
# Only a sighting *after* the fix is a regression. The window still reaches back over
|
||||
# sessions that predate it, and flagging those made the fix look undone — worse, it
|
||||
# opened findings whose whole evidence is about behaviour already dealt with.
|
||||
if last_seen > previous.get("applied", {}).get("at", "")[:10]:
|
||||
regression_of = previous["id"]
|
||||
else:
|
||||
stale_after_fix = True
|
||||
|
||||
was_rejected = item["pattern"] in rejected_patterns
|
||||
repeats = occurrences >= 2 and sessions >= 2
|
||||
silenced = was_rejected or stale_after_fix
|
||||
status = STATUS_OPEN if (repeats or regression_of) and not silenced else STATUS_WATCH
|
||||
|
||||
results.append(
|
||||
Finding(
|
||||
id=_new_id(used_ids),
|
||||
status=status,
|
||||
created=today,
|
||||
last_seen=last_seen,
|
||||
pattern=item["pattern"],
|
||||
severity=item["severity"],
|
||||
diagnosis=item["diagnosis"],
|
||||
evidence=evidence,
|
||||
occurrences=occurrences,
|
||||
sessions_affected=sessions,
|
||||
proposal=item["proposal"],
|
||||
patch=item["patch"],
|
||||
regression_of=regression_of,
|
||||
history=history,
|
||||
)
|
||||
)
|
||||
used_ids.add(results[-1].id)
|
||||
return results
|
||||
|
||||
|
||||
def supersede(existing: list[dict[str, Any]], merged: list[Finding]) -> list[dict[str, Any]]:
|
||||
"""Drop the watch/open records that the new findings fold in, then append the new ones."""
|
||||
folded = {record_id for finding in merged for entry in finding.history for record_id in [entry.split(":")[-1]]}
|
||||
kept = [record for record in existing if record["id"] not in folded]
|
||||
return kept + [finding.to_json() for finding in merged]
|
||||
|
||||
|
||||
def _rate_line(rate: float, previous: float | None) -> str:
|
||||
"""How often already-known patterns still recur — the one number that says whether this pays off.
|
||||
|
||||
Without a trend the report is only a restatement of findings.jsonl; with it, an applied fix
|
||||
that did nothing becomes visible in the next run.
|
||||
"""
|
||||
trend = f" (previous run {previous:.1f})" if previous is not None else ""
|
||||
return f"Known patterns: {rate:.1f} occurrences / 100 sessions{trend}."
|
||||
|
||||
|
||||
def _window_line(stats: dict[str, Any]) -> str:
|
||||
"""What the findings below actually cover — without it the report reads as if it saw everything."""
|
||||
scope = f"from {stats['window_from'][:10]}" if stats.get("window_from") else "from the cursor, no window"
|
||||
return f"Window: {scope}, batches {stats['batches']}/{stats.get('batches_total', stats['batches'])}."
|
||||
|
||||
|
||||
def _seen_line(finding: Finding) -> str:
|
||||
"""First and last sighting: `occurrences` is cumulative, so the count alone hides staleness.
|
||||
|
||||
"Last seen" is the newest date in the evidence, not `created` — a record refiled every night
|
||||
reported today's date whatever the evidence behind it said.
|
||||
"""
|
||||
first = finding.history[0].split(":")[0] if finding.history else finding.created
|
||||
return f"first seen {first}, last seen {finding.last_seen}"
|
||||
|
||||
|
||||
def _is_stale(finding: Finding, stats: dict[str, Any]) -> bool:
|
||||
"""Nothing in the run's window backs this finding any more, so no run will refresh it either."""
|
||||
window_from = str(stats.get("window_from") or "")[:10]
|
||||
return bool(window_from) and finding.last_seen < window_from
|
||||
|
||||
|
||||
def render_report(merged: list[Finding], stats: dict[str, Any], previous_rate: float | None, when: datetime) -> str:
|
||||
"""The dated results/ report — the human-readable record behind the Telegram one-liner."""
|
||||
lines = [
|
||||
f"# Self-reflection {when:%Y-%m-%d}",
|
||||
"",
|
||||
f"Analysed {stats['sessions']} sessions in {stats['batches']} batches. Findings: {len(merged)} "
|
||||
f"({sum(1 for f in merged if f.status == STATUS_OPEN)} to review, "
|
||||
f"{sum(1 for f in merged if f.status == STATUS_WATCH)} watched).",
|
||||
"",
|
||||
_window_line(stats),
|
||||
_rate_line(stats["repeat_per_100"], previous_rate),
|
||||
"",
|
||||
]
|
||||
if not merged:
|
||||
lines.append("Nothing to report.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
for finding in sorted(merged, key=lambda f: (f.status != STATUS_OPEN, SEVERITIES.index(f.severity) * -1)):
|
||||
flag = " — REGRESSION" if finding.regression_of else ""
|
||||
if _is_stale(finding, stats):
|
||||
flag += " — STALE"
|
||||
lines += [
|
||||
f"## {finding.id} · `{finding.pattern}` [{finding.status}/{finding.severity}]{flag}",
|
||||
"",
|
||||
finding.diagnosis,
|
||||
"",
|
||||
f"**Occurrences:** {finding.occurrences}× in {finding.sessions_affected} sessions · {_seen_line(finding)}",
|
||||
"",
|
||||
"**Evidence:**",
|
||||
]
|
||||
lines += [
|
||||
f"- `{item.get('session', '?')}` {item.get('when', '')} — {item.get('excerpt', '')}".rstrip(" —")
|
||||
for item in finding.evidence
|
||||
]
|
||||
lines += ["", f"**Proposal:** {finding.proposal}", ""]
|
||||
if finding.patch:
|
||||
lines += [
|
||||
f"**Patch:** `{finding.patch['file']}`",
|
||||
"",
|
||||
"```diff",
|
||||
*[f"- {line}" for line in finding.patch["old_text"].splitlines()],
|
||||
*[f"+ {line}" for line in finding.patch["new_text"].splitlines()],
|
||||
"```",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
async def _resolve_findings(
|
||||
bot: Nanobot, session_key: str, prompt: str, workspace: Path, session_count: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Ask the agent for findings, retrying with validator feedback in the same session.
|
||||
|
||||
A provider failure is counted separately from a malformed answer. The two look alike from
|
||||
here — nanobot hands back `Error calling LLM: …` as the reply — but blaming the model for
|
||||
text it never wrote burns a validator attempt and sends it a nonsensical correction.
|
||||
"""
|
||||
fingerprint = _git_fingerprint(workspace)
|
||||
message = prompt
|
||||
last_error: FindingsError = FindingsError("- no attempt was made")
|
||||
attempt = 0
|
||||
llm_errors = 0
|
||||
|
||||
while attempt < MAX_ATTEMPTS:
|
||||
result = await bot.run(message, session_key=session_key)
|
||||
if result.stop_reason == "error" or result.error:
|
||||
llm_errors += 1
|
||||
detail = (result.error or result.content or "unknown").splitlines()[0]
|
||||
print(f"[llm-error {llm_errors}/{MAX_LLM_ERROR_RETRIES}] {detail}", file=sys.stderr)
|
||||
if llm_errors > MAX_LLM_ERROR_RETRIES:
|
||||
raise ReflectError(f"model unavailable: {detail}")
|
||||
# The failed turn left an error message and the whole digest in that session; a fresh
|
||||
# key re-asks the original question instead of paying for the poisoned history.
|
||||
session_key = f"{session_key}r{llm_errors}"
|
||||
message = prompt
|
||||
continue
|
||||
|
||||
attempt += 1
|
||||
content = result.content or ""
|
||||
print(f"[attempt {attempt}/{MAX_ATTEMPTS}] {len(content)} chars", file=sys.stderr)
|
||||
|
||||
if _git_fingerprint(workspace) != fingerprint:
|
||||
raise ReflectError("the agent modified the workspace during an analysis run — nothing filed")
|
||||
|
||||
try:
|
||||
parsed, problems = parse_findings(content, session_count)
|
||||
except FindingsError as error:
|
||||
last_error = error
|
||||
print(f"[attempt {attempt}/{MAX_ATTEMPTS}] rejected:\n{error}", file=sys.stderr)
|
||||
message = RETRY_PROMPT.format(errors=error)
|
||||
continue
|
||||
|
||||
for problem in problems:
|
||||
print(f"[attempt {attempt}/{MAX_ATTEMPTS}] {problem}", file=sys.stderr)
|
||||
return parsed
|
||||
|
||||
raise ReflectError(f"after {MAX_ATTEMPTS} attempts: {str(last_error).splitlines()[0].lstrip('- ')}")
|
||||
|
||||
|
||||
def build_prompt(digest: str, known_patterns: str, session_count: int) -> str:
|
||||
return GOAL.format(
|
||||
count=session_count,
|
||||
max_findings=MAX_FINDINGS_PER_BATCH,
|
||||
known_patterns=known_patterns,
|
||||
digest=digest,
|
||||
)
|
||||
|
||||
|
||||
def _open_bot(preset: str):
|
||||
"""Deferred import so a dry run works on a machine without nanobot-ai installed."""
|
||||
from nanobot import Nanobot # ty: ignore[unresolved-import]
|
||||
|
||||
return Nanobot.from_config(model_preset=preset)
|
||||
|
||||
|
||||
async def _run(workspace: Path, args: argparse.Namespace, now: datetime) -> tuple[str, int]:
|
||||
state = _load_state(workspace)
|
||||
# Cursor is the floor, window the ceiling: nothing is analysed twice and nothing older than
|
||||
# the window is analysed at all, so a run always describes recent behaviour and the backlog
|
||||
# cannot starve it. Sessions the window skips are skipped for good — the cursor moves past them.
|
||||
window_from = f"{now - timedelta(days=args.window_days):%Y-%m-%dT%H:%M:%S}" if args.window_days else ""
|
||||
since = "" if args.all else max(str(state.get("cursor") or ""), window_from)
|
||||
paths = collect_sessions(workspace / "sessions", since=since)
|
||||
batches = list(iter_batches(paths, args.budget_chars))
|
||||
if args.max_batches:
|
||||
batches = batches[: args.max_batches]
|
||||
if not batches:
|
||||
return "", 0
|
||||
|
||||
existing = _load_findings(workspace)
|
||||
known_before = {record["pattern"] for record in existing}
|
||||
bot = None if args.dry_run else _open_bot(MODEL_PRESET)
|
||||
runs = state.setdefault("runs", [])
|
||||
previous_rate = runs[-1].get("repeat_per_100") if runs else None
|
||||
stats: dict[str, Any] = {
|
||||
"at": f"{now:%Y-%m-%d %H:%M}",
|
||||
"window_from": window_from,
|
||||
"sessions": 0,
|
||||
"batches": 0,
|
||||
"batches_total": len(batches),
|
||||
"open": 0,
|
||||
"watch": 0,
|
||||
"repeat_per_100": 0.0,
|
||||
}
|
||||
# Keyed by pattern: a pattern found in five batches is one finding, and only the last fold
|
||||
# of it is the complete one. Appending every batch's copy inflated the report headings, the
|
||||
# open/watch counts and the Telegram number to several times what the store actually holds.
|
||||
merged: dict[str, Finding] = {}
|
||||
session_count = 0
|
||||
repeat_occurrences = 0
|
||||
deadline_seconds = args.deadline_minutes * 60
|
||||
started_at = time.monotonic()
|
||||
|
||||
def commit(batch: list[SessionDigest]) -> None:
|
||||
"""Persist everything analysed so far, after every batch.
|
||||
|
||||
A run that gets cut short — by the deadline, the hard timeout or a dead provider — must
|
||||
keep the batches it did finish and leave the cursor past them. Doing this once at the end
|
||||
meant a timeout threw away finished work and left the cursor still pointing at it.
|
||||
"""
|
||||
_write_findings(workspace, existing)
|
||||
stats.update(
|
||||
sessions=session_count,
|
||||
batches=stats["batches"] + 1,
|
||||
open=sum(1 for f in merged.values() if f.status == STATUS_OPEN),
|
||||
watch=sum(1 for f in merged.values() if f.status == STATUS_WATCH),
|
||||
repeat_per_100=round(repeat_occurrences / session_count * 100, 1) if session_count else 0.0,
|
||||
)
|
||||
report = workspace / RESULTS_REL / f"{now:%Y-%m-%d}_reflect.md"
|
||||
report.parent.mkdir(parents=True, exist_ok=True)
|
||||
report.write_text(render_report(list(merged.values()), stats, previous_rate, now), encoding="utf-8")
|
||||
# Batches run oldest-first, so the newest start inside one is a safe cursor. Never move it
|
||||
# backwards: an `--all` run walks the whole backlog and must not undo the nightly progress
|
||||
# if it dies halfway.
|
||||
newest = max((item.started for item in batch), default="")
|
||||
state["cursor"] = max(newest, str(state.get("cursor") or ""))
|
||||
if stats["batches"] == 1:
|
||||
runs.append(stats)
|
||||
_save_state(workspace, state)
|
||||
|
||||
for index, batch in enumerate(batches):
|
||||
if index and time.monotonic() - started_at > deadline_seconds:
|
||||
print(f"deadline: stopping after {index}/{len(batches)} batches", file=sys.stderr)
|
||||
break
|
||||
|
||||
digest = "\n\n".join(item.text for item in batch)
|
||||
session_count += len(batch)
|
||||
prompt = build_prompt(digest, _known_patterns(existing), len(batch))
|
||||
if args.dry_run:
|
||||
target = workspace / "tmp" / f"reflect-batch.{index:03d}.md"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(prompt, encoding="utf-8")
|
||||
print(f"dry-run: {target} ({len(prompt) / 1000:.0f} kB)", file=sys.stderr)
|
||||
continue
|
||||
|
||||
session_key = f"reflect:{now:%Y%m%d-%H%M%S}-{index}"
|
||||
parsed = await _resolve_findings(bot, session_key, prompt, workspace, len(batch))
|
||||
# Newly observed occurrences only — the cumulative counts on `merged` carry earlier
|
||||
# runs with them and would inflate the rate every time a pattern is folded in.
|
||||
repeat_occurrences += sum(item["occurrences"] for item in parsed if item["pattern"] in known_before)
|
||||
batch_findings = merge_findings(existing, parsed, f"{now:%Y-%m-%d}")
|
||||
existing = supersede(existing, batch_findings)
|
||||
merged.update({finding.pattern: finding for finding in batch_findings})
|
||||
commit(batch)
|
||||
|
||||
if args.dry_run:
|
||||
return "", session_count
|
||||
|
||||
pending = len(batches) - stats["batches"]
|
||||
opened = [f for f in merged.values() if f.status == STATUS_OPEN]
|
||||
if not opened:
|
||||
# Silence is how the starved cursor went unnoticed for three nights: a run that cannot
|
||||
# keep up finds nothing new, and used to say nothing about the backlog it left behind.
|
||||
if pending:
|
||||
scope = window_from[:10] or "the cursor"
|
||||
return f"🔍 reflect: 0 findings, {pending} batches left (window from {scope}).", session_count
|
||||
return "", session_count
|
||||
|
||||
patchable = sum(1 for f in opened if f.patch)
|
||||
regressions = sum(1 for f in opened if f.regression_of)
|
||||
text = f"🔍 reflect: {len(opened)} findings to review ({patchable} with a patch)"
|
||||
if regressions:
|
||||
text += f", {regressions} regressions"
|
||||
done = stats["batches"]
|
||||
progress = "" if done == len(batches) else f" Analysed {done}/{len(batches)} batches."
|
||||
return f"{text}.{progress} Type /reflect.", session_count
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--dry-run", action="store_true", help="write prompts to tmp/, call no model")
|
||||
parser.add_argument("--all", action="store_true", help="ignore the cursor and re-read everything (dry runs only)")
|
||||
parser.add_argument(
|
||||
"--window-days",
|
||||
type=int,
|
||||
default=DEFAULT_WINDOW_DAYS,
|
||||
help=f"analyse only sessions from the last N days (default {DEFAULT_WINDOW_DAYS}, 0 = from the cursor)",
|
||||
)
|
||||
parser.add_argument("--budget-chars", type=int, default=DEFAULT_BUDGET_CHARS)
|
||||
parser.add_argument("--max-batches", type=int, default=0, help="stop after N batches (0 = no limit)")
|
||||
parser.add_argument(
|
||||
"--deadline-minutes",
|
||||
type=int,
|
||||
default=DEFAULT_DEADLINE_MINUTES,
|
||||
help="start no new batch past this many minutes (0 = one batch per run)",
|
||||
)
|
||||
parser.add_argument("--workspace", type=Path, help="override the configured workspace (for dry runs)")
|
||||
args = parser.parse_args(argv)
|
||||
if args.all and not args.dry_run:
|
||||
# merge_findings folds by pattern and sums the counts, so re-reading analysed sessions
|
||||
# inflates every occurrence count. It is a debugging view, not a way to recount.
|
||||
parser.error("--all re-reads sessions already counted and would inflate the counts; use it with --dry-run")
|
||||
|
||||
now = datetime.now()
|
||||
# A dry run must work without the server config, so it can be checked from a dev machine.
|
||||
config = {} if (args.dry_run and args.workspace) else _config()
|
||||
workspace = args.workspace or _workspace(config)
|
||||
|
||||
try:
|
||||
message, sessions = asyncio.run(asyncio.wait_for(_run(workspace, args, now), timeout=TIMEOUT_SECONDS))
|
||||
failed = False
|
||||
except TimeoutError:
|
||||
message = f"🔍 reflect: ERROR — timed out after {TIMEOUT_SECONDS // 60} min, finished batches are saved."
|
||||
sessions, failed = 0, True
|
||||
except ReflectError as error:
|
||||
message, sessions = f"🔍 reflect: ERROR — {error}.", 0
|
||||
failed = True
|
||||
except Exception as error: # noqa: BLE001 — an unattended job must report, not just die
|
||||
traceback.print_exc()
|
||||
message, sessions = f"🔍 reflect: ERROR — {type(error).__name__}: {error}.", 0
|
||||
failed = True
|
||||
|
||||
if not message:
|
||||
print(f"reflect_auto: {sessions} sessions, nothing to report", file=sys.stderr)
|
||||
return 0
|
||||
if failed:
|
||||
print(f"reflect_auto: {message}", file=sys.stderr)
|
||||
if args.dry_run:
|
||||
return 1 if failed else 0
|
||||
|
||||
token, chat_id = _telegram_config(config)
|
||||
try:
|
||||
_send_telegram(message, token, chat_id)
|
||||
except Exception as error: # noqa: BLE001 — delivery failure must not hide the original outcome
|
||||
print(f"reflect_auto: telegram delivery failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user