From f77cc2dcfe1e702a667d4a3b982a9ca7c5651155 Mon Sep 17 00:00:00 2001 From: lachtan Date: Wed, 2 Sep 2026 10:36:37 +0200 Subject: [PATCH] upravy projektu a skillu --- projects/chata/memory.md | 4 + projects/chata/prompt.md | 13 + projects/chata/state.md | 28 + projects/life/memory.md | 1 + projects/life/state.md | 6 + projects/proxmox/memory.md | 3 +- projects/proxmox/state.md | 105 +++ skills/plan/SKILL.md | 11 +- skills/project/SKILL.md | 136 ++- skills/project/scripts/project_cli.py | 213 +++++ skills/project/tests/test_project_cli.py | 184 ++++ skills/reflect/README.md | 164 ++++ skills/reflect/SKILL.md | 205 +++++ skills/reflect/scripts/reflect_apply.py | 296 ++++++ skills/reflect/scripts/reflect_auto.py | 897 +++++++++++++++++++ skills/reflect/scripts/reflect_distill.py | 338 +++++++ skills/reflect/tests/test_reflect_apply.py | 381 ++++++++ skills/reflect/tests/test_reflect_auto.py | 701 +++++++++++++++ skills/reflect/tests/test_reflect_distill.py | 241 +++++ 19 files changed, 3875 insertions(+), 52 deletions(-) create mode 100644 projects/chata/memory.md create mode 100644 projects/chata/prompt.md create mode 100644 projects/chata/state.md create mode 100755 skills/project/scripts/project_cli.py create mode 100644 skills/project/tests/test_project_cli.py create mode 100644 skills/reflect/README.md create mode 100644 skills/reflect/SKILL.md create mode 100644 skills/reflect/scripts/reflect_apply.py create mode 100644 skills/reflect/scripts/reflect_auto.py create mode 100644 skills/reflect/scripts/reflect_distill.py create mode 100644 skills/reflect/tests/test_reflect_apply.py create mode 100644 skills/reflect/tests/test_reflect_auto.py create mode 100644 skills/reflect/tests/test_reflect_distill.py diff --git a/projects/chata/memory.md b/projects/chata/memory.md new file mode 100644 index 0000000..df7608f --- /dev/null +++ b/projects/chata/memory.md @@ -0,0 +1,4 @@ +- 2026-02-14: Založen projekt „chata" jako úložiště poznámek o chatě v Černých Voděradech (vlastní lachtan + žena Bětka) — úkoly, plány, návody (čištění vodárny apod.), co vyšlo/ne vyšlo. Uživatel bude i vyhledávat. Poznámka: Velké Losiny (barák mámy) jsou úplně jiný objekt, do projektu nepatří. +- 2026-09-01: ROZHODNUTÍ — nákup palivového dřeva do krbu s okenkem (dřevník 4 m³): koupit úplně suchý dub za 1 995 Kč/PRMs (sypaný metr), délka 33 cm. Důvody: cena pod trhem (suchý dub okolí Prahy 2 300–2 600 Kč/PRMs, Libeznice 2 590 Kč/PRMs; přepočet na rovný metr ~1 300 Kč/PRMr); čerstvé tvrdé dřevo by potřebovalo 2–3 roky sušení (letos by se topit nedalo, rozdíl jen 680 Kč za 4 m³); suchý dub levnější na jednotku tepla (~265 Kč/GJ) než suchý smrk (~360 Kč/GJ); krb s okenkem vyžaduje suché dřevo (mokré špiní sklo sazami, dehet v komíně) a dub hoří déle a klidněji než smrk (ten prská iskrami). Délka 33 cm ne 25 cm: dvě polínka za sebou do dřevníku, sedí i do kamen — těsnější uložení 25 cm je proti tomu irelevantní. Jednotky: 4 PRMs ≈ 2,4–2,6 rovných m³; na plné naplnění dřevníku ~6 PRMs (8 970 Kč). Kontrola při dodání: vlhkoměr dřeva (Ermenrich Wett MW20, ~450 Kč), dub ~15 % vlhkosti, měřit do čerstvého řezu na více polínech; cenu dopravy (nad 5 km symbolicky účtovaná) vyjasnit předem. +- 2026-09-01: Dřevo (suchý dub, 4 m³, 33 cm) OBJEDNÁNO — dodání až za měsíc, hlavní sezóna, vysoká poptávka. Mezitím ve starém dřevníku ~0,5 m³ zásoba. Vlhkoměr objednán k měření při dodání. +- 2026-09-01: OTEVŘENÁ OTÁZKA — zazimování chaty. Možnosti a rizika: (a) zazimovat (vypustit vodu) → obavy z náročného opětovného natlakování domácí vodárny na jaře (neprověřeno); (b) nezazimovat → riziko prasknutí vody jako loni (vytopilo sklep, zdi nasákly); (c) temperovat elektrickým kotlem → při větší zimě až ~300 Kč/den. Rozhodnutí zatím nepadlo. diff --git a/projects/chata/prompt.md b/projects/chata/prompt.md new file mode 100644 index 0000000..6b88a53 --- /dev/null +++ b/projects/chata/prompt.md @@ -0,0 +1,13 @@ +# Chata + +Úložiště všeho možného o chatě v Černých Voděradech — vlastnictví: lachtan a jeho žena Bětka. + +## Účel projektu +- Poznámky o chatě: co je potřeba udělat, co vyšlo a co nešlo, plány +- Provozní návody (např. čištění domácí vodárny) +- Cokoliv dalšího relevantního k chatě + +## Jak s tím pracovat +- Uživatel sem píše poznámky, agent je ukládá do `memory.md` (chronologicky) a `state.md` (aktuální stav) +- Uživatel bude v poznámkách vyhledávat — odpovídat plnými detaily, ne shrnutími +- Poznámky uživatel organizuje hashtagy (#chata) \ No newline at end of file diff --git a/projects/chata/state.md b/projects/chata/state.md new file mode 100644 index 0000000..98ea654 --- /dev/null +++ b/projects/chata/state.md @@ -0,0 +1,28 @@ +# Stav + +## Palivové dřevo + +Objednáno (2026-09-01): **suchý dub, 4 m³ (PRMs), délka 33 cm, 1 995 Kč/PRMs**. +Dodání až za měsíc — hlavní sezóna, vysoká poptávka. Do té doby vystačí +~0,5 m³ zásoba ve starém dřevníku. + +Na plné naplnění dřevníku by bylo potřeba ~6 PRMs (8 970 Kč); 4 PRMs ≈ 2,4–2,6 +rovných m³. + +**Při dodání zkontrolovat:** vlhkost vlhkoměrem (Ermenrich Wett MW20, objednán) +— dub má mít ~15 %, měřit do čerstvého řezu na více polínech. Cenu dopravy +(nad 5 km účtovaná symbolicky) vyjasnit předem. + +Zdůvodnění volby (cena pod trhem, suchý dub vs. čerstvé dřevo i vs. smrk, +požadavky krbu s okénkem, délka 33 vs. 25 cm) je v `memory.md` u záznamu +2026-09-01. + +## Otevřené otázky + +**Zazimování chaty — rozhodnutí zatím nepadlo.** Tři možnosti: + +- **Zazimovat (vypustit vodu)** — obava z náročného opětovného natlakování + domácí vodárny na jaře (neprověřeno). +- **Nezazimovat** — riziko prasknutí vody jako loni: vytopilo sklep, zdi + nasákly. +- **Temperovat elektrickým kotlem** — při větší zimě až ~300 Kč/den. diff --git a/projects/life/memory.md b/projects/life/memory.md index 70c55b5..48bed6a 100644 --- a/projects/life/memory.md +++ b/projects/life/memory.md @@ -16,3 +16,4 @@ salát, měli zavřeno → sushi (nasi goreng). Odpoledne chuť na sladké / něco „plnícího“; pravidlo „jen ovoce neomezeně“ → dal si jablko. Poznámka: chuť zůstává i po jablku. +- 2026-09-02: Potřebuje zajít na fyzio terapii — poradit s omezenou mobilitou pravého kyčle a zjistit, proč se pravá noha na rozdíl od levé tak snadno a často přetěžuje. Cíl: cvičit pravidelně, aspoň 2× týdně; protahování ještě častěji. diff --git a/projects/life/state.md b/projects/life/state.md index 29d294f..ebe6471 100644 --- a/projects/life/state.md +++ b/projects/life/state.md @@ -38,3 +38,9 @@ Denní vážení, vnímaná kondice, jestli mě něco bolí. - Pravidlo „jen ovoce neomezeně“ jako nouzová náhrada sladkého / svačiny. - Pozorování: po teplejším jídle (sushi) přichází chuť na sladké / zasytivé. - Otázka k řešení: jak tu chuť zastavit nebo nahradit něčím, co nezvedne váhu. + +## Mobilita a pohyb (2026-09-02) + +- **Fyzio terapie k zadání**: omezená mobilita pravého kyčle; proč se pravá noha + na rozdíl od levé tak snadno a často přetěžuje. +- **Cíl pravidelnosti**: cvičit aspoň 2× týdně; protahování častěji než doteď. diff --git a/projects/proxmox/memory.md b/projects/proxmox/memory.md index 970399b..0bddf90 100644 --- a/projects/proxmox/memory.md +++ b/projects/proxmox/memory.md @@ -5,4 +5,5 @@ - 2026-08-31: Základní kontext — dlouholetý provoz Proxmoxu (doma i v práci), zkušený Linux admin, preferuje CLI. RTX 4060 Ti/16 GB passthrough do VM. Dvě vícedenní školení Proxmox (základy + pokročilý). - 2026-08-31: Tři témata: (1) ZFS do hloubky, (2) PBS layout SSD rootfs + spiny + special vdev na SSD, (3) migrace Hyper-V → Proxmox na Dell Edge (2× 2 TB SSD) pro kamaráda — popsat a naskriptovat. - 2026-08-31: Dlouhodobé cíle: znovupoužití 2TB LUKS+LVM disku na atom serveru, rdiff-backup kontejner, záloha Incus kontejneru s Giteou (kritická data), zvážení PBS na atom serveru jako target pro hlavní Proxmox. -- 2026-08-31: Zvažuje migraci Gitey z Incusu na PVE (jednodušší zálohy přes PBS) + Gitea Actions runner (act_runner) jako další VM/kontejner. \ No newline at end of file +- 2026-08-31: Zvažuje migraci Gitey z Incusu na PVE (jednodušší zálohy přes PBS) + Gitea Actions runner (act_runner) jako další VM/kontejner. +- 2026-09-01: Rozhodnuto — Gitea migruje na PVE (atompivo), záloha přes PBS. Datastore: 2TB spin 5400 rpm, special vdev na SSD (nutnost kvůli GC na 5400 rpm; bez něj ~20+h denně, s ním minuty). Special vdev jako zvol nad root SSD poolem — SPOF akceptován, nelze odebrat při raidz topologii. Tuning: recordsize=1M, logbias=throughput, special_small_blocks=64K, chunk-order pro verify. PBS jako VM, celý disk passthrough, ARC RAM rezerva, chicken-egg řešen zfs send konfigurace mimo datastore. diff --git a/projects/proxmox/state.md b/projects/proxmox/state.md index 7131456..3d525e7 100644 --- a/projects/proxmox/state.md +++ b/projects/proxmox/state.md @@ -39,7 +39,112 @@ - **PBS na atom serveru?** — zvažuje, zda na atom serveru spustit Proxmox Backup Server jako zálohovací cíl pro hlavní Proxmox. +## Hostname / přejmenování nodu + +- **Název nodu = kernel hostname (`uname -n`)**, ne DNS. Změna DNS záznamu + Proxmoxu vůbec nevadí (ověřeno); změna `/etc/hostname` na nodu s VM + rozběhne systém — VM se nenašly, vše se chovalo divně. Pomohl revert. +- Root cause: pmxcfs drží konfigurace VM/kontejnerů, certifikáty a metadata + v `/etc/pve/nodes//`. Po změně hostname hledá pvedaemon konfigurace + pod novým názvem, který v pmxcfs neexistuje → VM "zmizí". +- **Správný postup** (oficiální wiki Renaming a PVE node): + 1. Doporučeno jen na **prázdném nodu** — pmxcfs neumí přejmenovat + neprázdné adresáře. + 2. Editovat `/etc/hostname` **a** `/etc/hosts` (všechny výskyty, hostname + na hlavní IP). Volitelně `/etc/mailname`, `/etc/postfix/main.cf`. + 3. V clusteru: upravit `n_0` v `/etc/pve/corosync.conf` přes + `pvecm edit corosync.conf` — ale v clusteru se přejmenování + **nedoporučuje**. + 4. Reboot. + 5. Ručně přenést `/etc/pve/nodes//` → `/` (strukturu + rekonstruovat, kopírovat po úrovních). + 6. Přenést RRD data: `/var/lib/rrdcached/db/pve2-{node,storage}/` + → ``, starý smazat. +- **Moudro:** u nodu s běžícími VM stojí přejmenování tolik ruční práce, + že je čistší postavit node pod novým názvem a VM migrnout. Revert + hostname vše opraví — pmxcfs data nemaže, jen je "nezobrazí". + ## Studijní materiály — ZFS (uložené v bookmark skillu, db/bookmark.sqlite) - [ZFS 101 (Jim Salter, Ars Technica)](https://arstechnica.com/information-technology/2020/05/zfs-101-understanding-zfs-storage-and-performance/) — zpools, vdevy, RAIDZ vs mirror, ashift, CoW, ZIL/SLOG, komprese, ARC. Plný text archivován. - [ZFS vs RAID benchmark (Ars Technica)](https://arstechnica.com/gadgets/2020/05/zfs-versus-raid-eight-ironwolf-disks-two-filesystems-one-winner/) — 8× Ironwolf, mirrors vs RAID10, RAIDZ2 vs RAID6, vliv recordsize; výkon roste s počtem vdevů, ne disků. Plný text archivován. + +## Atompivo — hardware kontext + +- Server **atompivo**: Proxmox, stavěný dřív, než bylo jasné finální využití. +- **1 TB SSD** — celý disk na root fs nad ZFS (žádná volná partition). +- **2 TB spin disk** — 5400 rpm, starší, aktuálně šifrovaná data (LUKS z předchozího + Debianu); plánovaná přestavba na datastore. Další 2 TB spin disk k dispozici. + +## PBS datastore na atompivo — layout a tuning + +- **PBS poběží jako VM** na atompivu (oficiálně podporované, běžné). Požadavky: + celý disk passthrough pro spiny i pro special vdev (virtio-scsi), rezervovat + dost RAM pro ARC (řádově 4-8 GB), atom = ingest spíš CPU-bound (SHA256/zstd). +- **Chicken-egg**: PBS VM zálohuje node, na kterém sama běží — umět PBS postavit + znovu z ničeho; konfiguraci datastore + vzdump PBS VM držet mimo datastore + (např. `zfs send` z root poolu na jiný stroj). +- **Datastore pool** na 2TB spiny (5400 rpm), tuning hned od založení: + - `recordsize=1M` + `logbias=throughput` (doloženo na Proxmox foru: ingest + 200 → 900 MB/s jen touto změnou) + - **special vdev na SSD** — na 5400 rpm **nutnost, ne nice-to-have**: + GC je metadata workload (readdir+stat nad miliony chunků); bez special vdevu + ~20+ h denně, s ním minuty až desítky minut. GC defaultně běží denně. + - `special_small_blocks=64K` — obsah malých chunků také na SSD; hlídat + zaplnění special vdevu (`zpool list -v`), řádově desítky GB. + - **`proxmox-backup-debug chunk-order`** — povinný na 5400 rpm: bez něj + random read ~1-2 MB/s efektivně → verify týdny; s ním sekvenční read + ~100-130 MB/s → ~5-6 h na 2 TB. Pustit po naplnění datastore a po + velkých změnách znovu. +- **Special vdev = zvol nad SSD root poolem** (žádná volná partition): + - Overhead nested CoW (write amplification ~2-3× na malých writzech, + dvojité journaling vrstvy) — pro metadata na SSD irelevantní. + - **SPOF**: ztráta SSD = ztráta celého poolu i se zdravými spiny. + Akceptováno (GC bolí denně, ztráta SSD je hypotetický jednorázový event). + - **Nelze později odebrat**, pokud top-level vdevy jsou raidz nebo ashifty + neschodují (`zpool remove` restrikce) — topologii naplánovat předem; + mirror spinů = odstranitelné, raidz = ne. + - Special vdev lze naopak přidat kdykoliv (`zpool add -o ... -special`) + — alternativní postup: založit bez něj, změřit, přidat. + +## Gitea → PVE migrace a zálohy + +- Rozhodnuto: **Gitea migruje z Incusu na Proxmox** — standardní PBS záloha + VM/kontejneru místo ad-hoc zálohování Incusu. +- Zálohovací cíl: **PBS na atompivu** (datastore viz výše). + +## Šifrování — co Proxmox umí a co ne + +- **PVE installer neumí šifrovaný root FS** — žádná LUKS volba. LUKS root = + ruční instalace Debianu + PVE na něj (komunitní návody: mr-manuel + luks-encryption-manual-tpm-ssh-unlock, cmspam/proxmox-uki-installer). + Feature request v Bugzille otevřený. +- **Hardware TPM pro host unlock: PVE neumí, ale jde DIY** — TPM2 sealing přes + `systemd-cryptenroll --tpm2` (nebo clevis) na Debian vrstvě; pro smysluplné + PCR7 sealing vhodné/žádoucí Secure Boot. PVE samo TPM spravuje jen pro + **guesty** (vTPM, Windows 11) — to je jiná věc než host unlock. +- **ZFS native encryption = experimentální v PVE docs** — známé limity: + replikace encrypted datasets (Bugzilla 2350), checksum errory u snapshotů + a ZVOLů (openzfs#11688). **Žádná TPM integrace** — jen passphrase/keyfile, + `zfs load-key` ručně nebo systemd unit. +- **Čistá cesta pro data disky: ZFS nad LUKS** (`/dev/mapper/...` jako vdev + zařízení) — osvědčený NAS pattern, TPM2 auto-unlock možný přes LUKS. + Pozor: `cryptsetup --sector-size 4096` pro ashift alignment. +- **Atom caveat**: zkontrolovat AES-NI (`lscpu | grep aes`) a přítomnost + TPM čipu — bez AES-NI LUKS throughput na atomu výrazně padá. + +## Atompivo — reinstalace vs. stávající layout + +- Problém: celý SSD patří rpoolu → žádná volná partition pro special vdev + (jen zvol nad rpoolem) ani pro budoucí potřeby. +- **Reinstalace overkill?** Pro samotný special vdev ano (zvol stačí), ale + node se právě přestavuje (data na spinu k přestavbě, Gitea se stěhuje + dovnitř) — reinstalace nikdy nebude levnější než teď. +- Installer umí advanced options (menší HDSize → volné místo na SSD). +- Bonusy reinstalace: swap na partition místo rpool/swap ZVOLu (PVE wiki + explicitně varuje před swap na zvolu), čistá partition pro special vdev + (jedna vrstva ZFS, žádný nested CoW, nezávislost na rpoolu), LVM rezerva. +- **Otevřená otázka: PBS VM vs. PBS v LXC** — LXC by posunul ZFS pool + datastore na host (žádný disk passthrough, special vdev = host partition/ + zvol, LUKS pod ZFS řešený na hostu vč. TPM). Váže se na rozhodnutí o + šifrování a layoutu. diff --git a/skills/plan/SKILL.md b/skills/plan/SKILL.md index 5865f61..d9dc3c3 100644 --- a/skills/plan/SKILL.md +++ b/skills/plan/SKILL.md @@ -50,14 +50,14 @@ otherwise proceed. ``` # - ## Kontext + ## Context Why this change — the problem, what prompted it, the intended outcome. - ## Postup + ## Steps Numbered steps. Name the files to touch. Reference reusable code found in phase 1 with its path. - ## Ověření + ## Verification How to test the change end-to-end (run it, run tests, check behavior). ``` @@ -65,8 +65,9 @@ otherwise proceed. ## 4. Approval (replaces ExitPlanMode — over chat) -Stop and ask: `Plán uložen do workspace/plans/<slug>.md. Schvaluješ? Mám ho -vykonat teď?` Then wait. Mutate nothing on your own. +Stop and ask for approval: say the plan was saved to +`workspace/plans/<slug>.md`, and ask whether to execute it now. Then wait. +Mutate nothing on your own. - Approved + execute now → drop the read-only discipline and execute the plan in this conversation. diff --git a/skills/project/SKILL.md b/skills/project/SKILL.md index 8e0c182..2b7a8b9 100644 --- a/skills/project/SKILL.md +++ b/skills/project/SKILL.md @@ -4,9 +4,9 @@ description: > Switch between named ongoing projects, each with its own persistent context, carried forward across turns until the user switches or ends it. Triggers on: "project X", "switch to project X", "we're working on X", - "end project" / "no project". For a long-lived, named work context — not a - single task to plan and execute, and not a short or global fact to - remember. + "list projects", "end project" / "no project". For a long-lived, named + work context — not a single task to plan and execute, and not a short or + global fact to remember. --- # Project @@ -19,17 +19,28 @@ separate from every other project and from the agent's general memory. `workspace/projects/<slug>/`: -- `prompt.md` — project-specific context/instructions, read in full whenever - the project becomes active +- `prompt.md` — **what doesn't change**: purpose, goals, scope, constraints, + how the user wants to work on this. Read in full whenever the project + becomes active. - `memory.md` — append-only chronological log of decisions and history -- `state.md` — living document: the current state/synthesis of the project, - edited in place, not appended to +- `state.md` — **what changes**: the living synthesis of where the project is + now. Progress, current approach, open questions, "what's next". Edited in + place, never appended to. - `artifacts/` — generated files (documents, code, data exports, reports) +If you're about to write anything time-varying into `prompt.md` (a "current +status" or "next steps" section), put it in `state.md` instead. + **Which project is active is tracked purely through conversation memory — -nothing is persisted to disk for that.** Once a project is chosen, keep -treating it as active for the rest of this conversation; don't re-read its -files on every subsequent turn once they're already in context. +nothing is persisted to disk for that.** + +## Script + +All project file operations go through one script, run from the workspace root: + +`uv run skills/project/scripts/project_cli.py <subcommand>` + +It owns the entry date and the append hygiene so they can't be guessed wrong. ## Activation @@ -37,20 +48,19 @@ Triggered by "project X" / "switch to project X" / "we're working on X": 1. Turn `X` into a kebab-case directory name (like picking a slug for a plan file — no need to spell out an algorithm, just pick something sensible). -2. **`workspace/projects/<slug>/` already exists** → read `prompt.md`, - `memory.md`, and `state.md` in full, then briefly confirm what's active - and, if `prompt.md` or `state.md` has content, what context you loaded. -3. **It doesn't exist, but the name is a close match to one or more existing - projects** (case-insensitive) → ask which one was meant. Don't guess, and - don't create a near-duplicate of an existing project. -4. **It doesn't exist and there's no close match** → this skill only works - with projects that already exist without asking. Ask the user whether to - start a new project with that name. Only on explicit yes, create - `prompt.md`, `memory.md`, `state.md` (all empty) and `artifacts/`. Never - create a project just because a trigger phrase was said. +2. Run `project_cli.py activate <slug>`. On success it prints `prompt.md`, + `memory.md` and `state.md` (creating any that are missing), so no separate + reads are needed. Then briefly confirm what's active and, if `prompt.md` or + `state.md` has content, what context you loaded. +3. **Exit 1 means no such project.** The error lists the existing ones. If one + is a close match (case-insensitive), ask which was meant — don't guess, and + don't create a near-duplicate. +4. **No close match** → ask the user whether to start a new project with that + name. Only on an explicit yes, run `project_cli.py new <slug>`. Never create + a project just because a trigger phrase was said. **Already active:** if the same project is already active in this -conversation, don't re-create or re-read anything — just continue. +conversation, don't re-run activation — just continue. ## Staying active @@ -58,6 +68,11 @@ Once a project is active, keep applying its `prompt.md` instructions and `state.md` context for the rest of the conversation, until the user switches or ends it. +**If the file contents are no longer in your context** (a long conversation +gets compacted, and the loaded files can drop out of it while the memory that +a project is active stays), run `activate` again. Never answer from a faded +recollection of `prompt.md` or `state.md`. + If a long gap or an ambiguous reference makes it unclear whether the project is still the right context (e.g. the conversation has clearly moved to an unrelated topic), ask rather than silently carrying it forward or silently @@ -74,55 +89,84 @@ dropping it. ## Writing to memory.md -Append only — never rewrite or reorder existing entries. One entry per -decision, dead end, or noteworthy piece of history: +**Only ever through the script** — never `edit_file` or `write_file`. Pass the +text on stdin with a quoted heredoc so quotes and apostrophes survive verbatim: ``` -- YYYY-MM-DD: <terse entry, reformulated, not verbatim> +uv run skills/project/scripts/project_cli.py log <slug> <<'NOTE' +<entry text> +NOTE ``` -Write an entry when a decision is made, a dead end is found, or a fact -central to the project's ongoing context emerges — not for routine -back-and-forth. When in doubt whether something is memory-worthy, prefer not -writing it; `memory.md` is for what a future conversation needs to pick up -the thread, not a transcript. +The script prepends today's date and guarantees the entry starts on its own +line. Write the entry in the user's language, reformulated, not verbatim. + +Write an entry when a decision is made, a dead end is found, or a fact central +to the project's ongoing context emerges — not for routine back-and-forth. When +in doubt, **write it**: losing something the user told you is worse than an +entry that turns out to be unremarkable. Entry length is not limited — capture +the reasoning behind a decision, not just its outcome. + +**Correcting an entry:** `memory.md` is append-only, so never rewrite history. +Log a new entry starting with `correction:` that states what was wrong. ## Maintaining state.md -Unlike `memory.md`, `state.md` is edited in place: surgically update the -relevant section when the project's current state or understanding has -moved on enough that the old text would mislead a reader. It answers "where -is this now", not "what happened" — old content gets replaced, not appended -to. If it's still empty and the project has accumulated enough context to -synthesize, offer to draft it. +Edited in place with `edit_file`: surgically update the relevant section when +the project's current state or understanding has moved on enough that the old +text would mislead a reader. It answers "where is this now", not "what +happened" — old content gets replaced, not appended to. + +**If `state.md` is empty and `memory.md` has content, draft it** from that +history and show the user the result. Don't just offer. Likewise, if +`prompt.md` still holds a time-varying section, move it here. + +## Answering questions about a project + +Answers that live in project files come **from the files**, not from memory: +read or `grep` them. If a project is active, its files are the first place to +look; with no project active, `grep` across `workspace/projects/` to find where +something was written. + +## What goes where + +While a project is active, a fact tied to that project goes into its +`memory.md` via the script. A durable fact about the user in general — one that +would still matter with no project active — goes to the normal memory path +instead. ## Artifacts Files generated while working on the project (documents, code, data exports, -reports) go in `workspace/projects/<slug>/artifacts/` instead of scattered -elsewhere. Name them descriptively; no numbering or index file needed at +reports) go in `workspace/projects/<slug>/artifacts/`, created the first time +one is written. Name them descriptively; no numbering or index file needed at this scale. +## Growth + +`memory.md` grows without limit and is **never compacted, archived or +summarized**. When it gets long, `activate` simply stops printing all of it and +says where the rest is — read or `grep` the file directly for older context. +Never delete, shorten or rewrite stored content to save space. + ## Listing -**"list projects" / "which projects exist":** list the directory names under -`workspace/projects/`. If none exist, say so. +**"list projects" / "which projects exist":** run `project_cli.py list`. It +prints each project with its file sizes; `(!)` marks an empty `state.md`. ## Edge cases - "project" with no name → ask which project. -- The project directory exists but one or more of `prompt.md` / `memory.md` - / `state.md` is missing (e.g. created by hand) → create the missing - file(s) empty, don't error. +- Missing `prompt.md` / `memory.md` / `state.md` (e.g. directory made by hand) + → `activate` creates them, don't error. - Deleting or renaming a project is out of scope for this skill — point the user at `workspace/projects/<slug>/` to do it by hand. ## Rules -- This skill's body is English; reply to the user in their own language. +- This skill's body is English; reply to the user in their own language, and + write project files in the user's language too. - Never fabricate project content — `prompt.md`, `memory.md`, and `state.md` only grow from what the user actually said or what actually happened. - Never create a new project without the user's explicit confirmation. - Don't force a project context onto an unrelated request. -- The Dream memory processor must not touch `workspace/projects/` — it is - outside the memory and skills Dream curates. diff --git a/skills/project/scripts/project_cli.py b/skills/project/scripts/project_cli.py new file mode 100755 index 0000000..7259a34 --- /dev/null +++ b/skills/project/scripts/project_cli.py @@ -0,0 +1,213 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""project_cli.py — deterministic file operations for the /project skill. + +The agent must never hand-edit projects/<slug>/memory.md: dates get invented and +appends get joined onto the previous line when the anchor is guessed. This script +owns both — the date comes from the system clock, the newline is guaranteed. + +Nothing here ever shortens, rewrites or deletes stored content. `activate` may +omit older memory entries from its *output* when the whole project would exceed +the tool-result limit, but the files on disk are left untouched. +""" + +import argparse +import json +import os +import sys +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +# workspace/skills/project/scripts/project_cli.py -> parents[3] = workspace root. +WORKSPACE = Path(__file__).resolve().parents[3] + +TIMEZONE = ZoneInfo("Europe/Prague") +PROJECT_FILES = ("prompt.md", "memory.md", "state.md") + +# Tool results above `maxToolResultChars` (16000, server config.json) are offloaded +# to a file the agent then has to read back in pieces. Stay under it with a margin. +MAX_OUTPUT_CHARS = 14_400 + + +def projects_dir() -> Path: + """Root of the project store; PROJECTS_DIR overrides it for tests.""" + override = os.environ.get("PROJECTS_DIR") + return Path(override) if override else WORKSPACE / "projects" + + +def project_path(slug: str) -> Path: + return projects_dir() / slug + + +def existing_slugs() -> list[str]: + root = projects_dir() + if not root.is_dir(): + return [] + return sorted(entry.name for entry in root.iterdir() if entry.is_dir()) + + +def read_file(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except FileNotFoundError: + return "" + + +def ensure_project_files(directory: Path) -> None: + """Create any missing project file as empty — a hand-made directory must work.""" + directory.mkdir(parents=True, exist_ok=True) + for name in PROJECT_FILES: + path = directory / name + if not path.exists(): + path.write_text("", encoding="utf-8") + + +def format_size(size: int) -> str: + if size < 1024: + return f"{size}B" + return f"{size / 1024:.1f}K" + + +def fit_memory(memory: str, budget: int, slug: str) -> str: + """Drop the oldest entries from the *output* until it fits the budget. + + The file itself is never modified — the note tells the agent where the rest is. + """ + if len(memory) <= budget: + return memory + lines = memory.splitlines(keepends=True) + kept: list[str] = [] + used = 0 + for line in reversed(lines): + if used + len(line) > budget: + break + kept.append(line) + used += len(line) + kept.reverse() + omitted = len(lines) - len(kept) + note = ( + f"[… {omitted} older entries not shown, full log: " + f"projects/{slug}/memory.md — read it when you need older context]\n" + ) + return note + "".join(kept) + + +def cmd_activate(slug: str) -> int: + directory = project_path(slug) + if not directory.is_dir(): + slugs = existing_slugs() + listing = ", ".join(slugs) if slugs else "(none)" + print(f"No such project: {slug}. Existing: {listing}", file=sys.stderr) + return 1 + + ensure_project_files(directory) + prompt = read_file(directory / "prompt.md") + memory = read_file(directory / "memory.md") + state = read_file(directory / "state.md") + + # prompt.md and state.md always go out whole; only memory.md gives ground. + overhead = len(prompt) + len(state) + 200 + memory_out = fit_memory(memory, max(MAX_OUTPUT_CHARS - overhead, 0), slug) + + sections = [ + f"### prompt.md\n{prompt}", + f"### memory.md\n{memory_out}", + f"### state.md\n{state}", + ] + print("\n\n".join(section.rstrip() + "\n" for section in sections), end="") + if not state.strip(): + print("\n[!] state.md is empty") + return 0 + + +def cmd_log(slug: str, text: str | None) -> int: + directory = project_path(slug) + if not directory.is_dir(): + slugs = existing_slugs() + listing = ", ".join(slugs) if slugs else "(none)" + print(f"No such project: {slug}. Existing: {listing}", file=sys.stderr) + return 1 + + body = text if text is not None else sys.stdin.read() + body = body.strip() + if not body: + print("Nothing to log (empty input).", file=sys.stderr) + return 1 + + today = datetime.now(TIMEZONE).date().isoformat() + entry = f"- {today}: {body}\n" + + memory_file = directory / "memory.md" + existing = read_file(memory_file) + # Guarantee the new entry starts on its own line, whatever the file ends with. + separator = "" if not existing or existing.endswith("\n") else "\n" + with memory_file.open("a", encoding="utf-8") as handle: + handle.write(separator + entry) + + print(json.dumps({"appended": entry.rstrip("\n")}, ensure_ascii=False)) + return 0 + + +def cmd_list() -> int: + slugs = existing_slugs() + if not slugs: + print("(no projects yet)") + return 0 + width = max(len(slug) for slug in slugs) + for slug in slugs: + directory = project_path(slug) + sizes = [] + for name in PROJECT_FILES: + path = directory / name + size = path.stat().st_size if path.is_file() else 0 + label = name.removesuffix(".md") + flag = " (!)" if name == "state.md" and size == 0 else "" + sizes.append(f"{label} {format_size(size)}{flag}") + print(f"{slug:<{width}} " + " ".join(sizes)) + return 0 + + +def cmd_new(slug: str) -> int: + directory = project_path(slug) + if directory.exists(): + print(f"Project already exists: {slug}", file=sys.stderr) + return 1 + ensure_project_files(directory) + print(json.dumps({"created": slug, "files": list(PROJECT_FILES)}, ensure_ascii=False)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="File operations for the /project skill") + sub = parser.add_subparsers(dest="command", required=True) + + activate = sub.add_parser("activate", help="Print a project's three files") + activate.add_argument("slug") + + log = sub.add_parser("log", help="Append a dated entry to memory.md") + log.add_argument("slug") + log.add_argument( + "--text", default=None, help="Entry text; if omitted, read from stdin" + ) + + sub.add_parser("list", help="List projects with file sizes") + + new = sub.add_parser("new", help="Create an empty project") + new.add_argument("slug") + + args = parser.parse_args() + if args.command == "activate": + return cmd_activate(args.slug) + if args.command == "log": + return cmd_log(args.slug, args.text) + if args.command == "list": + return cmd_list() + return cmd_new(args.slug) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/project/tests/test_project_cli.py b/skills/project/tests/test_project_cli.py new file mode 100644 index 0000000..aa96f04 --- /dev/null +++ b/skills/project/tests/test_project_cli.py @@ -0,0 +1,184 @@ +"""Tests for project_cli.py — deterministic file operations for the /project skill.""" + +import io +import json +import sys +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import project_cli # noqa: E402 + + +@pytest.fixture +def projects(tmp_path, monkeypatch): + root = tmp_path / "projects" + root.mkdir() + monkeypatch.setenv("PROJECTS_DIR", str(root)) + return root + + +def make_project(projects, slug, prompt="", memory="", state=""): + directory = projects / slug + directory.mkdir() + (directory / "prompt.md").write_text(prompt, encoding="utf-8") + (directory / "memory.md").write_text(memory, encoding="utf-8") + (directory / "state.md").write_text(state, encoding="utf-8") + return directory + + +def today(): + return datetime.now(ZoneInfo("Europe/Prague")).date().isoformat() + + +# -- log --------------------------------------------------------------------- + + +def test_log_appends_with_today_date(projects, capsys): + directory = make_project(projects, "chata") + + assert project_cli.cmd_log("chata", "Dřevo objednáno") == 0 + + memory = (directory / "memory.md").read_text(encoding="utf-8") + assert memory == f"- {today()}: Dřevo objednáno\n" + assert json.loads(capsys.readouterr().out)["appended"].startswith(f"- {today()}:") + + +def test_log_does_not_join_when_file_lacks_trailing_newline(projects): + directory = make_project(projects, "chata", memory="- 2026-09-01: první") + + project_cli.cmd_log("chata", "druhý") + + lines = (directory / "memory.md").read_text(encoding="utf-8").splitlines() + assert lines == ["- 2026-09-01: první", f"- {today()}: druhý"] + + +def test_log_creates_missing_memory_file(projects): + directory = projects / "chata" + directory.mkdir() + + assert project_cli.cmd_log("chata", "první") == 0 + assert (directory / "memory.md").read_text(encoding="utf-8").endswith("první\n") + + +def test_log_reads_stdin_verbatim(projects, monkeypatch): + directory = make_project(projects, "chata") + text = "Uvozovky „takhle\" a apostrof ' a \"tohle\"\ndruhý řádek" + monkeypatch.setattr(sys, "stdin", io.StringIO(text)) + + assert project_cli.cmd_log("chata", None) == 0 + + memory = (directory / "memory.md").read_text(encoding="utf-8") + assert memory == f"- {today()}: {text}\n" + + +def test_log_does_not_shorten_long_entry(projects): + directory = make_project(projects, "chata") + text = "x" * 3000 + + project_cli.cmd_log("chata", text) + + assert text in (directory / "memory.md").read_text(encoding="utf-8") + + +def test_log_rejects_empty_input(projects): + directory = make_project(projects, "chata", memory="- 2026-09-01: první\n") + + assert project_cli.cmd_log("chata", " ") == 1 + assert (directory / "memory.md").read_text(encoding="utf-8") == "- 2026-09-01: první\n" + + +def test_log_rejects_unknown_project(projects): + assert project_cli.cmd_log("neznamy", "text") == 1 + + +# -- activate ---------------------------------------------------------------- + + +def test_activate_prints_all_three_sections(projects, capsys): + make_project(projects, "chata", prompt="# Chata", memory="- 2026-09-01: a\n", state="stav") + + assert project_cli.cmd_activate("chata") == 0 + + out = capsys.readouterr().out + assert "### prompt.md\n# Chata" in out + assert "### memory.md\n- 2026-09-01: a" in out + assert "### state.md\nstav" in out + assert "[!] state.md is empty" not in out + + +def test_activate_creates_missing_files(projects, capsys): + directory = projects / "chata" + directory.mkdir() + + assert project_cli.cmd_activate("chata") == 0 + + assert all((directory / name).is_file() for name in project_cli.PROJECT_FILES) + + +def test_activate_flags_empty_state(projects, capsys): + make_project(projects, "chata", memory="- 2026-09-01: a\n") + + project_cli.cmd_activate("chata") + + assert "[!] state.md is empty" in capsys.readouterr().out + + +def test_activate_omits_oldest_entries_without_touching_disk(projects, capsys): + memory = "".join(f"- 2026-09-01: entry {i} {'x' * 200}\n" for i in range(200)) + directory = make_project(projects, "big", prompt="P", memory=memory, state="S") + size_before = (directory / "memory.md").stat().st_size + + assert project_cli.cmd_activate("big") == 0 + + out = capsys.readouterr().out + assert len(out) <= project_cli.MAX_OUTPUT_CHARS + assert "older entries not shown" in out + assert "entry 199" in out and "entry 0 " not in out + assert "### prompt.md\nP" in out and "### state.md\nS" in out + assert (directory / "memory.md").stat().st_size == size_before + + +def test_activate_rejects_unknown_project(projects, capsys): + make_project(projects, "chata") + + assert project_cli.cmd_activate("neznamy") == 1 + assert "chata" in capsys.readouterr().err + + +# -- list / new -------------------------------------------------------------- + + +def test_list_reports_sizes_and_flags_empty_state(projects, capsys): + make_project(projects, "chata", prompt="x" * 595, memory="y" * 2048) + make_project(projects, "life", prompt="a", memory="b", state="c") + + assert project_cli.cmd_list() == 0 + + lines = capsys.readouterr().out.splitlines() + assert "prompt 595B" in lines[0] and "memory 2.0K" in lines[0] + assert "state 0B (!)" in lines[0] + assert "(!)" not in lines[1] + + +def test_list_without_projects(projects, capsys): + assert project_cli.cmd_list() == 0 + assert capsys.readouterr().out.strip() == "(no projects yet)" + + +def test_new_creates_empty_files_without_artifacts(projects, capsys): + assert project_cli.cmd_new("novy") == 0 + + directory = projects / "novy" + assert all((directory / name).read_text(encoding="utf-8") == "" for name in project_cli.PROJECT_FILES) + assert not (directory / "artifacts").exists() + + +def test_new_refuses_existing_project(projects): + make_project(projects, "chata", memory="- 2026-09-01: a\n") + + assert project_cli.cmd_new("chata") == 1 + assert (projects / "chata" / "memory.md").read_text(encoding="utf-8") == "- 2026-09-01: a\n" diff --git a/skills/reflect/README.md b/skills/reflect/README.md new file mode 100644 index 0000000..fb2ee80 --- /dev/null +++ b/skills/reflect/README.md @@ -0,0 +1,164 @@ +# reflect — jak to funguje + +Skill hledá v mých vlastních session logách **opakující se chyby**, pojmenuje je a navrhne +opravu. Nálezy pak procházíš ty, jeden po druhém, a rozhoduješ, co se použije. +**Sám od sebe nikdy nic nezmění.** + +## Dva režimy + +```text +ANALÝZA — denně 03:30, bez tebe REVIEW — jen když napíšeš /reflect +cron → reflect_auto.py ├─ vezme 1 otevřený nález + ├─ destiluje session z okna ├─ ukáže diagnózu, důkazy, návrh, diff + ├─ LLM tah → nálezy ├─ čeká na tvoje rozhodnutí + ├─ zapíše do findings.jsonl ├─ aplikuje → git commit → audit + └─ Telegram (jen když je co) └─ další nález +NEEDITUJE NIC edituje jen to, co schválíš +``` + +## Nálezy jsou o nedávném chování, ne o celé historii + +Běh se dívá jen na **posledních 21 dní** (`--window-days`). Není to úspora — je to +podmínka, aby nález něco znamenal: korpus má 3,5 měsíce a jedna dávka 200 kB, takže +dohánění celé historie po jedné dávce za noc znamenalo, že cursor tři noci stál na +konci května a nálezy z něj se předkládaly jako aktuální. + +Dvě hranice, každá jinak: + +| | Co dělá | +|---|---| +| **cursor** | podlaha — co se jednou analyzovalo, se neanalyzuje znovu (jinak by se zdvojily počty) | +| **okno** | strop — co je starší než 21 dní, se přeskočí a cursor to mine natrvalo | + +Kdybys někdy potřeboval archeologii, vrať `cursor` ve `state.json` a spusť +`--window-days 0`. + +## Proč tě to neotravuje každý den + +Denně přibude jen pár session — v nich se vzor nepozná, jedna chyba je náhoda. Proto: + +| Kdy | Status | Telegram | +|---|---|---| +| vzor poprvé, 1 výskyt | `watch` | ne, jen se počítá | +| vzor podruhé (≥2× a ve ≥2 session) | `open` | ano | +| vzor už jednou opravený se vrátí **po** opravě | `open` + **regrese** | ano | +| vzor už jednou opravený, ale důkazy jsou starší než oprava | `watch` | ne | +| vzor jsi zamítl | `watch` napořád | ne, už nikdy | + +Zamítnutí je rozhodnutí, ne odklad — zamítnutý vzor se znovu neotevře. + +## Co ti /reflect nabídne + +| Napíšeš | Stane se | +|---|---| +| `ok` / `aplikuj` | patch se použije a commitne | +| `uprav: <text>` | přepíšeš návrh vlastními slovy, ukáže se nový diff | +| `přeskoč` | nález zůstane otevřený na příště | +| `zamítni` | nález se zavře natrvalo | +| `konec` | konec review | + +Nález, který od analýzy patch nedostal (většina), si ho složí až při review: agent napíše +návrh do JSON a nechá ho ověřit (`--set-patch`). Skript ho **nejdřív ověří proti souboru +a teprve pak uloží**, takže nepoužitelný pokus ve `findings.jsonl` nezůstane — a agent do +store nesahá vůbec. Pak ti ukáže diff a čeká na `ok` jako u každého jiného patche. + +Vždy jen **jeden** nález najednou. Nálezy se číslují `1..N` podle pořadí, ne podle +interního id. + +## Kde co leží + +| Cesta | Co | +|---|---| +| `reflect/findings.jsonl` | všechny nálezy, jeden JSON na řádek | +| `reflect/state.json` | kam se došlo (cursor), použité okno + statistiky běhů | +| `results/<datum>_reflect.md` | plný report jednoho běhu + okno, které pokryl, a kolik výskytů na 100 session mají už známé vzory | +| `log/reflect.log` | audit — každé rozhodnutí, i zamítnutí a přeskočení | + +**„Naposledy" u nálezu je nejnovější datum v důkazech**, ne datum, kdy se záznam +naposledy přepsal. Když je starší než okno posledního běhu, je nález **zastaralý**: +analýza tam už nedohlédne, takže se sám nikdy neobnoví — spíš než patch si zaslouží +zamítnutí. `/reflect` ti to u něj řekne. + +**Počty v nálezu (`7× ve 4 session`) jsou kumulativní součet napříč běhy** — sečtené +z toho, co model napočítal v jednotlivých oknech, ne měření nad celým korpusem. Skript +u nich hlídá jen to, co hlídat umí: vzor nemůže zasáhnout víc session, než kolikrát +nastal, ani víc, než kolik jich v dávce vůbec bylo. + +## Co se o tvém rozhodnutí zapíše + +| Rozhodnutí | Do `findings.jsonl` | Do `log/reflect.log` | +|---|---|---| +| složení patche | `patch` + `patch_drafted_at` (= složeno při review, ne modelem) | `DRAFTED <id> [vzor] <soubor>` | +| `ok` | `applied: {at, sha, file}`, `patch` = návrh modelu | `APPLIED <id> [vzor] <soubor> — <sha>` | +| `uprav:` | navíc `applied.new_text` = tvoje verze (návrh modelu zůstává v `patch`) | `APPLIED-EDITED …` | +| `zamítni` | `rejected: {at, reason}` — **důvod je povinný** | `REJECTED <id> [vzor] — <důvod>` | +| `přeskoč` | `skipped: {count, last}`, status zůstává `open` | `SKIPPED <id> [vzor] ×N` | + +Povinný důvod u zamítnutí není otravování: je to jediná zpětná vazba na kvalitu analýzy. +Z prvních osmi nálezů jsi čtyři zamítl — bez důvodů se z toho čísla nedá poznat, co +v analytickém promptu změnit. + +Na dotaz „co jsem už rozhodl" ti to `/reflect` vypíše z logu včetně revert příkazu. + +## Jak vrátit změnu zpět + +Workspace je git, takže každá aplikovaná oprava je samostatný commit: + +```bash +git revert <sha> +``` + +SHA najdeš v `log/reflect.log` nebo u nálezu ve `findings.jsonl` (`applied.sha`). + +Změna žije **jen na serveru** — do trackovacího repa (`src/nanobot`) se musí dotáhnout +zvlášť, jinak ji příští `rsync` skillu přepíše zpátky. + +## Ruční spuštění + +```bash +cd ~/.nanobot/workspace + +# normální běh (to dělá cron) +~/.local/bin/uv run --script skills/reflect/scripts/reflect_auto.py + +# jiné okno než výchozích 21 dní +~/.local/bin/uv run --script skills/reflect/scripts/reflect_auto.py --window-days 7 + +# jen prompty do tmp/, nevolat model; s --all přes celý korpus +~/.local/bin/uv run --script skills/reflect/scripts/reflect_auto.py --dry-run --all + +# jak velký je korpus a na kolik dávek vyjde (nevolá model) +~/.local/bin/uv run --script skills/reflect/scripts/reflect_distill.py --stats > /dev/null +``` + +`--all` ignoruje cursor, takže by znovu přečetl už spočítané session a **nafoukl jim +počty**. V ostrém běhu ho skript odmítne — je jen na `--dry-run`. + +**Přerušený běh o hotovou práci nepřijde.** Nálezy i cursor se zapisují po každé dávce +a nová dávka nezačne po 20 minutách (`--deadline-minutes`). S 21denním oknem to vyjde +zpravidla na jednu dávku, takže se tenhle strop ani neuplatní; kdyby zbyla nezpracovaná +dávka, Telegram to řekne i když nálezy nejsou žádné. + +Plná cesta k `uv` je tu proto, že v neinteraktivním SSH není v `PATH`. Crontab si +`PATH` nastavuje sám, takže tam stačí `uv run …`. + +## Záruky + +Nejsou to sliby v promptu, ale kód: + +- **Analýza (`reflect_auto.py`) nemá v sobě žádnou cestu k zápisu do cizího souboru.** + Navíc se před a po tahu porovná `git status` celého workspace — kdyby agent přesto něco + zapsal, nálezy se zahodí. +- **Editaci dělá výhradně `reflect_apply.py`, vždy jeden nález.** Agent soubor needituje + sám. Skript odmítne patch, jehož původní text v souboru není nebo je tam vícekrát — + nehádá, kam patřil. Odmítne i nález, který není `open` (tj. nebyl ti předložen). +- **Do `findings.jsonl` píše taky jen `reflect_apply.py`.** I patch složený při review jde + přes něj (`--set-patch`) a projde stejnou kontrolou; agent auditní stopu needituje. +- **Commituje se jen ten jeden dotčený soubor** (`git add -- <file>`), nikdy `git add -A`. + Rozdělaná práce Dreamu a jiných skillů se do commitu nedostane; když je rozdělaný přímo + ten soubor, udělá se nejdřív checkpoint commit, aby byl revert přesný. +- Skill vynechává vlastní session, takže neanalyzuje sám sebe. +- **Každé rozhodnutí nechá záznam**, i to, které nic nezmění: zamítnutí s důvodem, + přeskočení s počítadlem, `uprav:` odděleně od původního návrhu modelu. + +Ověřeno testy v `tests/` — včetně toho, že `git revert` vrátí soubor do původního stavu. diff --git a/skills/reflect/SKILL.md b/skills/reflect/SKILL.md new file mode 100644 index 0000000..071cbd9 --- /dev/null +++ b/skills/reflect/SKILL.md @@ -0,0 +1,205 @@ +--- +name: reflect +description: > + Review findings from your own self-diagnosis and apply approved fixes, one at a time. + Triggers ONLY on the explicit `/reflect` command. Never infer this skill from + conversational mentions of reflection, findings, self-improvement, or "what did you + learn" — it can edit your own configuration, so it must never start by accident. +--- + +# Reflect + +Interactive review of findings produced by the unattended analysis run +(`scripts/reflect_auto.py`, daily via cron). You show the user one finding at a time, +wait for a decision, and apply only what they approve. + +Answer in the language the user writes in. + +## STOP gates — read before anything else + +1. **This skill never runs the analysis.** Analysis is a cron script. If the user asks + for a fresh analysis, tell them to run + `~/.local/bin/uv run --script skills/reflect/scripts/reflect_auto.py` — do not + distil sessions or diagnose patterns yourself in this turn. +2. **Never edit anything without approval of that specific finding.** Not "the user + seemed positive earlier", not "finding 3 is obviously right". One `ok` approves + exactly one finding. +3. **One finding per step.** Never present two findings at once, never apply a batch, + even if the user says "apply everything" — in that case apply the first, show the + result, and continue to the next. +4. **Never edit a target file yourself — always through `scripts/reflect_apply.py`.** + That script refuses a patch whose original text is missing or ambiguous, and commits + only the file it touched. If it refuses, relay the reason; never hand-edit around it. + **`reflect/findings.jsonl` is never edited by hand either — not with a one-off script, + not with a heredoc, never.** It is the audit trail; every write to it belongs to + `reflect_apply.py`. Drafting a patch during the review goes through `--set-patch`. + +## Data + +| Path | What | +|---|---| +| `reflect/findings.jsonl` | one JSON object per line — the findings store | +| `reflect/state.json` | cursor and per-run statistics (owned by the script) | +| `results/<date>_reflect.md` | full report of a run | +| `log/reflect.log` | append-only audit of every decision — applied, rejected, skipped | + +A finding has: `id`, `status`, `created`, `pattern`, `severity`, `diagnosis`, +`last_seen`, `evidence`, `occurrences`, `sessions_affected`, `proposal`, optional `patch` +(`file` / `old_text` / `new_text`), optional `patch_drafted_at` (the patch was drafted +during a review, not proposed by the analysis), optional `regression_of`, optional `history`, +and once decided one of `applied` (`at` / `sha` / `file`, plus `new_text` and +`edited_by_user` when the user rewrote it), `rejected` (`at` / `reason`), `skipped` +(`count` / `last`). + +Statuses: `watch` (seen once, not worth the user's attention yet) · `open` (waiting for +review) · `applied` · `rejected`. + +Two things about the numbers, both of which you must not overstate to the user: + +- A run analyses a **window** of recent sessions, not the whole history. `state.json` + holds the window it used; the dated report repeats it in its header. +- `occurrences` and `sessions_affected` are **cumulative across runs** — the sum of what + the model counted in each slice, not a figure anyone measured over the whole corpus. + `last_seen` is the newest date in the evidence, `history[0]` when the pattern was first + filed, `created` only when the record was last rewritten. Older records may lack + `last_seen`; fall back to `created` for those. + +## Procedure + +### 1. Load + +Read `reflect/findings.jsonl`. Take the records with `status: open`, sorted by severity +(`high`, `medium`, `low`), then by `last_seen` and then by `occurrences`, all descending. +`occurrences` alone would let a stale pattern with a large cumulative count outrank a +fresh one. + +A finding whose `last_seen` is older than the window of the latest run (`state.json`, +`window_from`) is **stale**: no run looks that far back any more, so nothing will refresh +it. Say so when you present it — it is a candidate for rejection rather than a patch. + +Assign **display IDs 1..N** over that sorted list, computed fresh each time. The user +refers to findings by these short numbers; the internal `id` stays the key in the store +and in the audit log, and is never what you ask the user to type. + +If there are none: say so, mention how many `watch` findings are being tracked, and stop. + +### 2. Present one finding + +Show exactly one, in this shape: + +```text +[1/4] retry-without-diagnosis · medium · 7× in 4 sessions +first seen 2026-09-01, last seen 2026-09-02 · deferred 2× already +<diagnosis> + +Evidence: +- websocket_e5a6aa… (2026-07-11) — web_fetch → ERROR 403 ×4 +- … + +Proposal: <proposal> +``` + +Write the labels in the user's language, not necessarily as shown here. + +The second line carries what the count alone hides: `history[0]` for when the pattern +was first filed, `last_seen` for when it last actually occurred, and `skipped.count` when +the user has already deferred it. Drop the skip part when there is none; a finding deferred +several times is worth saying so about, because rejecting it is cleaner than a list that +keeps re-presenting it. + +Mark a finding with `regression_of` clearly as a **regression** — this pattern was fixed +before and came back after the fix. Mark a stale finding (see step 1) as such too, and +say what it means: the evidence predates the current window, so the pattern may well be +gone already. + +If the finding has a `patch`, get the diff from the script — never by reading the file and +judging for yourself, that is the duplication gate 4 exists to prevent: + +```bash +uv run --script skills/reflect/scripts/reflect_apply.py --id <internal id> --check +``` + +It changes nothing and prints the diff; show the user what it printed. Exit code 2 means the +patch no longer applies — relay the printed reason and offer only *skip* / *reject*. + +Then ask for a decision and wait. + +### 3. Accept a decision + +| Input | Meaning | +|---|---| +| `ok`, `apply` | apply this finding's patch | +| `edit: <text>` | the user rewrites `new_text`; show the new diff and ask again | +| `skip` | record the deferral, leave it `open`, move to the next | +| `reject` | **ask why first**, then close it for good — it never opens again | +| `stop` | end the review | + +Accept the equivalents in whatever language the user writes in — the words above are the +meanings, not a required vocabulary. + +Rejection needs a reason and the script will not take it without one. Ask for it in one +short question and pass the user's own words through — half of the first findings were +rejected, and that number only says something about the analysis if the reasons are on +the record. Do not invent a reason, and do not talk the user out of rejecting. + +A finding without a `patch` cannot be applied. Offer to draft one, and file it with +`--set-patch` — **never by writing to `reflect/findings.jsonl` yourself** (gate 4): + +```bash +# {"file": "SOUL.md", "old_text": "<copied character-for-character>", "new_text": "<the fix>"} +uv run --script skills/reflect/scripts/reflect_apply.py --id <internal id> --set-patch tmp/patch.json +``` + +`old_text` must be copied from the current file and occur in it exactly once — copy it, +never retype it, and watch the quotation marks. The script verifies the patch *before* +storing it, so a refusal (exit 2) leaves the finding exactly as it was; relay the reason +and draft again. On success it prints the diff — show that, then ask for the `ok`. It +changes no file and decides nothing, so applying still needs step 4. + +### 4. Apply (only after `ok`) + +**Never edit the file yourself.** `scripts/reflect_apply.py` does it, and it is what +enforces the gates — it refuses a stale or ambiguous patch, commits only the touched +file, writes the audit line and updates the store. Editing by hand would bypass all of it. + +```bash +uv run --script skills/reflect/scripts/reflect_apply.py --id <internal id> +``` + +- The `--id` is the finding's internal `id`, not the display number you showed the user. +- User rewrote the text (`edit:`)? Write their version to a temp file and pass + `--new-text-file <path>`. +- *skip* → `--skip`. Changes no file and no status; it only counts the deferral. +- *reject* → `--reject --reason "<user's words>"`. It changes no file. +- Exit code 2 means refused: relay the printed reason and move on. Do not work around it, + do not edit the file to make the patch fit. + +On success the script prints the commit SHA and the revert command. Pass that on, then +continue with the next finding. + +### 5. Close + +When the user stops or the list is exhausted, summarise: how many applied, skipped, +rejected, and how many remain `open`. If anything was applied, remind the user that the +change lives only on the server and should be pulled back into the tracking repo. + +## Decision history + +Asked what was already decided, or what a past decision changed: read `log/reflect.log` +— it is the source of truth for the list, one line per decision, oldest first. Then fill +each line in from `findings.jsonl`: + +- applied → the `patch` (the model's proposal), `applied.new_text` when the user rewrote + it, and `revert: git revert <applied.sha>`. Use `git show <sha>` for the real diff. +- rejected → `rejected.reason`. Records decided before reasons existed carry a flat + `rejected_at` and no reason; say the reason is missing rather than guessing one. +- skipped → `skipped.count`, still open. + +Do not reconstruct this list from memory or from the findings store alone. + +## When a finding is wrong + +Findings come from an LLM reading its own logs and can be plain wrong. That is expected — +rejection is a normal outcome, not a failure. If several findings in a row are noise, say +so plainly; that is a signal the analysis prompt needs tuning, and it is worth telling the +user rather than working through a list of nonsense. diff --git a/skills/reflect/scripts/reflect_apply.py b/skills/reflect/scripts/reflect_apply.py new file mode 100644 index 0000000..e83cb31 --- /dev/null +++ b/skills/reflect/scripts/reflect_apply.py @@ -0,0 +1,296 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""reflect_apply.py — apply one approved finding, or refuse. + +The guarantees around self-modification live here rather than in SKILL.md prose, because +a soft instruction is not a gate: the agent could read "verify old_text occurs exactly +once" and still edit on a near miss. This script cannot. It applies exactly one finding, +only when the patch still matches, and it commits only the file it touched. + +The skill calls it after the user approves a specific finding. It never decides anything +on its own — no finding is selected, ranked or approved here. + +It is also the audit trail of the review, so every decision leaves a record: why a finding +was rejected, that a patch was applied with the user's own wording rather than the model's, +and how often a finding has been skipped without being decided. + + reflect_apply.py --id f7a2 apply that finding + reflect_apply.py --id f7a2 --check print the diff, change nothing + reflect_apply.py --id f7a2 --reject --reason mark rejected with the user's reason + reflect_apply.py --id f7a2 --skip count a deferral, decide nothing + reflect_apply.py --id f7a2 --new-text-file patch with user-edited replacement text + reflect_apply.py --id f7a2 --set-patch file a patch drafted during the review +""" + +from __future__ import annotations + +import argparse +import difflib +import json +import subprocess +import sys +from contextlib import suppress +from datetime import datetime +from pathlib import Path + +WORKSPACE = Path(__file__).resolve().parents[3] +FINDINGS_REL = "reflect/findings.jsonl" +LOG_REL = "log/reflect.log" + +STATUS_OPEN = "open" +STATUS_APPLIED = "applied" +STATUS_REJECTED = "rejected" + +PATCH_KEYS = ("file", "old_text", "new_text") + + +class ApplyError(Exception): + """A refusal, phrased for the agent to relay to the user.""" + + +def _git(workspace: Path, *args: str) -> str: + result = subprocess.run(["git", *args], cwd=workspace, capture_output=True, text=True, check=True) + return result.stdout.strip() + + +def _load(workspace: Path) -> list[dict]: + path = workspace / FINDINGS_REL + if not path.exists(): + raise ApplyError(f"{FINDINGS_REL} does not exist — run the analysis first") + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def _save(workspace: Path, records: list[dict]) -> None: + path = workspace / FINDINGS_REL + body = "\n".join(json.dumps(record, ensure_ascii=False) for record in records) + path.write_text(body + "\n", encoding="utf-8") + + +def _audit(workspace: Path, line: str) -> None: + log = workspace / LOG_REL + log.parent.mkdir(parents=True, exist_ok=True) + with log.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + + +def _find(records: list[dict], finding_id: str) -> dict: + for record in records: + if record["id"] == finding_id: + return record + raise ApplyError(f"no finding with id {finding_id}") + + +def _resolve_target(workspace: Path, relative: str) -> Path: + """Reject anything that would land outside the workspace, symlinks included.""" + target = (workspace / relative).resolve() + if not target.is_relative_to(workspace.resolve()): + raise ApplyError(f"{relative} resolves outside the workspace") + if not target.is_file(): + raise ApplyError(f"{relative} does not exist") + return target + + +def check_patch(workspace: Path, record: dict, new_text: str | None = None) -> tuple[Path, str, str]: + """Return (target, current content, patched content), or raise with the reason.""" + if record["status"] != STATUS_OPEN: + raise ApplyError(f"finding {record['id']} is {record['status']}, only open findings can be applied") + patch = record.get("patch") + if not patch: + raise ApplyError(f"finding {record['id']} carries no patch — nothing to apply") + + target = _resolve_target(workspace, patch["file"]) + content = target.read_text(encoding="utf-8") + occurrences = content.count(patch["old_text"]) + if occurrences == 0: + raise ApplyError(f"the original text is no longer in {patch['file']} — the patch does not apply") + if occurrences > 1: + raise ApplyError(f"the original text occurs {occurrences}× in {patch['file']} — too ambiguous to apply") + + replacement = patch["new_text"] if new_text is None else new_text + if replacement == patch["old_text"]: + raise ApplyError("the replacement is identical to the original — nothing to change") + return target, content, content.replace(patch["old_text"], replacement, 1) + + +def _diff(relative: str, current: str, patched: str) -> str: + return "\n".join( + difflib.unified_diff( + current.splitlines(), patched.splitlines(), fromfile=relative, tofile=relative, lineterm="" + ) + ) + + +def _parse_patch_file(path: Path) -> dict[str, str]: + """Read the drafted patch, refusing anything the store would not accept as one.""" + try: + patch = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ApplyError(f"{path} is not readable JSON: {error}") from error + if not isinstance(patch, dict) or sorted(patch) != sorted(PATCH_KEYS): + raise ApplyError(f"the patch must be a JSON object with exactly {list(PATCH_KEYS)}") + if not all(isinstance(patch[key], str) for key in PATCH_KEYS): + raise ApplyError("the patch fields must be strings") + return {key: patch[key] for key in PATCH_KEYS} + + +def set_patch(workspace: Path, finding_id: str, patch: dict[str, str], now: datetime) -> str: + """File a patch drafted during the review, but only one that already applies. + + A finding the analysis left without a patch used to force the agent to edit + findings.jsonl with an ad-hoc script — an LLM writing straight into the audit trail, + and a failed attempt leaving an unapplicable patch behind. Verifying the candidate + before anything is written is what makes that impossible: a patch that does not apply + never reaches the store at all. + """ + records = _load(workspace) + record = _find(records, finding_id) + # check_patch already refuses a non-open finding, resolves the path and counts the + # occurrences of old_text — run it on a candidate copy so nothing is written until it passes. + _, current, patched = check_patch(workspace, {**record, "patch": patch}) + + record["patch"] = patch + record["patch_drafted_at"] = f"{now:%Y-%m-%d %H:%M}" + _save(workspace, records) + + _audit(workspace, f"{now:%Y-%m-%d %H:%M} DRAFTED {finding_id} [{record['pattern']}] {patch['file']}") + return f"drafted a patch for {finding_id} on {patch['file']} — it applies cleanly, nothing written yet\n" + _diff( + patch["file"], current, patched + ) + + +def apply_finding(workspace: Path, finding_id: str, new_text: str | None, now: datetime) -> str: + records = _load(workspace) + record = _find(records, finding_id) + target, original, patched = check_patch(workspace, record, new_text) + relative = record["patch"]["file"] + + # Commit unrelated work on this one file first, so the patch commit is only the patch. + if _git(workspace, "status", "--porcelain", "--", relative): + _git(workspace, "add", "--", relative) + _git(workspace, "commit", "-m", f"reflect: checkpoint before {finding_id}", "--", relative) + + target.write_text(patched, encoding="utf-8") + try: + _git(workspace, "add", "--", relative) + _git(workspace, "commit", "-m", f"reflect: {record['pattern']} ({finding_id})", "--", relative) + except subprocess.CalledProcessError: + # The caller reports a refusal and moves on, so "refused" has to mean nothing happened. + # Undoing the write is what makes that true; unstaging is best effort and must never + # replace the original error. + target.write_text(original, encoding="utf-8") + with suppress(subprocess.CalledProcessError): + _git(workspace, "restore", "--staged", "--", relative) + raise + sha = _git(workspace, "rev-parse", "--short", "HEAD") + + record["status"] = STATUS_APPLIED + record["applied"] = {"at": f"{now:%Y-%m-%d %H:%M}", "sha": sha, "file": relative} + # `patch` stays the model's proposal. Overwriting it with the user's rewrite destroyed the + # only record of what was suggested versus what was actually approved. + if new_text is not None: + record["applied"]["new_text"] = new_text + record["applied"]["edited_by_user"] = True + _save(workspace, records) + + verb = "APPLIED-EDITED" if new_text is not None else "APPLIED" + _audit(workspace, f"{now:%Y-%m-%d %H:%M} {verb} {finding_id} [{record['pattern']}] {relative} — {sha}") + + return f"applied {finding_id} to {relative}, commit {sha} (revert: git revert {sha})" + + +def reject_finding(workspace: Path, finding_id: str, reason: str, now: datetime) -> str: + """Close a pattern for good, with the reason on the record. + + The reason is mandatory because it is the only feedback on the analysis itself: half of the + first eight findings were rejected, and without knowing why, that number says nothing about + what to change in the analysis prompt. + """ + records = _load(workspace) + record = _find(records, finding_id) + if record["status"] != STATUS_OPEN: + raise ApplyError(f"finding {finding_id} is {record['status']}, not open") + record["status"] = STATUS_REJECTED + record["rejected"] = {"at": f"{now:%Y-%m-%d %H:%M}", "reason": reason} + _save(workspace, records) + + _audit(workspace, f"{now:%Y-%m-%d %H:%M} REJECTED {finding_id} [{record['pattern']}] — {reason}") + return f"rejected {finding_id} — this pattern will not open again" + + +def skip_finding(workspace: Path, finding_id: str, now: datetime) -> str: + """Count a deferral. Changes no file and no status — a skip is not a decision. + + Without it a finding skipped five times is indistinguishable from one never shown, so the + review keeps re-presenting it with nothing to say about the history. + """ + records = _load(workspace) + record = _find(records, finding_id) + if record["status"] != STATUS_OPEN: + raise ApplyError(f"finding {finding_id} is {record['status']}, not open") + count = record.get("skipped", {}).get("count", 0) + 1 + record["skipped"] = {"count": count, "last": f"{now:%Y-%m-%d %H:%M}"} + _save(workspace, records) + + _audit(workspace, f"{now:%Y-%m-%d %H:%M} SKIPPED {finding_id} [{record['pattern']}] ×{count}") + return f"skipped {finding_id} — still open, skipped {count}× so far" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--id", required=True, help="finding id from reflect/findings.jsonl") + parser.add_argument("--check", action="store_true", help="verify the patch still applies, change nothing") + parser.add_argument("--reject", action="store_true", help="mark the finding rejected, change no file") + parser.add_argument("--reason", help="why the finding was rejected — required with --reject") + parser.add_argument("--skip", action="store_true", help="count a deferral; leaves the finding open") + parser.add_argument("--new-text-file", type=Path, help="file holding replacement text edited by the user") + parser.add_argument("--set-patch", type=Path, help="JSON file with the drafted {file, old_text, new_text}") + parser.add_argument("--workspace", type=Path, default=WORKSPACE) + args = parser.parse_args(argv) + + if sum([args.check, args.reject, args.skip, bool(args.set_patch)]) > 1: + parser.error("--check, --reject, --skip and --set-patch are mutually exclusive") + if args.reject and args.new_text_file: + parser.error("--reject changes no file, so --new-text-file makes no sense with it") + if args.set_patch and args.new_text_file: + parser.error("--set-patch already carries new_text, so --new-text-file makes no sense with it") + if args.reject and not (args.reason or "").strip(): + parser.error("--reject needs --reason: ask the user why and pass it through") + if args.reason and not args.reject: + parser.error("--reason only records why a finding was rejected, so it needs --reject") + + now = datetime.now() + try: + if args.reject: + print(reject_finding(args.workspace, args.id, args.reason.strip(), now)) + return 0 + if args.skip: + print(skip_finding(args.workspace, args.id, now)) + return 0 + if args.set_patch: + print(set_patch(args.workspace, args.id, _parse_patch_file(args.set_patch), now)) + return 0 + + new_text = args.new_text_file.read_text(encoding="utf-8") if args.new_text_file else None + if args.check: + record = _find(_load(args.workspace), args.id) + _, current, patched = check_patch(args.workspace, record, new_text) + relative = record["patch"]["file"] + print(f"ok: patch applies cleanly to {relative}") + print(_diff(relative, current, patched)) + return 0 + + print(apply_finding(args.workspace, args.id, new_text, now)) + return 0 + except ApplyError as error: + print(f"refused: {error}", file=sys.stderr) + return 2 + except subprocess.CalledProcessError as error: + print(f"refused: git failed: {error.stderr.strip() or error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/reflect/scripts/reflect_auto.py b/skills/reflect/scripts/reflect_auto.py new file mode 100644 index 0000000..3700bcf --- /dev/null +++ b/skills/reflect/scripts/reflect_auto.py @@ -0,0 +1,897 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = ["nanobot-ai"] +# /// +"""reflect_auto.py — unattended analysis run for the /reflect skill. + +Distils new sessions, asks the agent to diagnose recurring mistakes, and files the +findings. **It never applies anything** — there is no edit path in this script at all. +Applying a fix happens only in the interactive `/reflect` review, one finding at a time, +after the user approves it. + +Two guards protect that boundary: + * the agent is told not to write, and a git fingerprint of the workspace is compared + before and after the turn — if the agent wrote anything, nothing is filed; + * a finding seen only once is filed as `watch` and stays silent. Telegram is notified + only once a pattern repeats, so a daily run does not mean a daily notification. + +Same external-script pattern as skills/compact-memory/scripts/compact_memory_auto.py: +fresh never-reused session_key, delivery straight to the Telegram Bot API, and the +message composed here rather than by the model. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import subprocess +import sys +import time +import traceback +import urllib.parse +import urllib.request +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING, Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from reflect_distill import DEFAULT_BUDGET_CHARS, SessionDigest, collect_sessions, iter_batches + +if TYPE_CHECKING: + from nanobot import Nanobot # ty: ignore[unresolved-import] + +# A digest prompt is tens of thousands of tokens, and prefilling it on glm-5.3:cloud overruns +# nanobot's 120s per-request default — whose retry then throws the finished prefill away and +# starts over. Set before nanobot is imported (`_open_bot` defers it); overridable from the shell. +os.environ.setdefault("NANOBOT_OPENAI_COMPAT_TIMEOUT_S", "600") +os.environ.setdefault("NANOBOT_LLM_TIMEOUT_S", "900") + +CONFIG = Path.home() / ".nanobot" / "config.json" +WORKSPACE_FALLBACK = Path.home() / ".nanobot" / "workspace" + +STATE_REL = "reflect/state.json" +FINDINGS_REL = "reflect/findings.jsonl" +RESULTS_REL = "results" + +FALLBACK_CHAT_ID = "8826147089" +MODEL_PRESET = "glm53" +# The soft deadline is what actually bounds a run: no new batch starts past it, and everything +# already analysed is on disk. TIMEOUT_SECONDS only catches a single batch that hangs. +DEFAULT_DEADLINE_MINUTES = 20 +# The cursor is the floor of a run, this is the ceiling: findings are meant to describe recent +# behaviour, and a months-deep backlog walked one batch a night never gets there (2026-09-02: +# the cursor sat on 05-29 while 3 runs of findings were presented as current). +DEFAULT_WINDOW_DAYS = 21 +TIMEOUT_SECONDS = 45 * 60 +MAX_ATTEMPTS = 3 +MAX_LLM_ERROR_RETRIES = 2 + +SEVERITIES = ("low", "medium", "high") +STATUS_WATCH = "watch" +STATUS_OPEN = "open" +STATUS_APPLIED = "applied" +STATUS_REJECTED = "rejected" +PATCH_KEYS = frozenset({"file", "old_text", "new_text"}) +FINDING_KEYS = frozenset( + {"pattern", "severity", "diagnosis", "evidence", "occurrences", "sessions_affected", "proposal", "patch"} +) +MAX_DIAGNOSIS_CHARS = 600 +MAX_PROPOSAL_CHARS = 400 +MAX_FINDINGS_PER_BATCH = 12 +# Folded findings keep evidence from earlier slices too, or a cumulative count is unauditable. +MAX_EVIDENCE = 6 +PATTERN_RE = re.compile(r"^[a-z][a-z0-9-]{2,48}$") +# `when` is a free string the model fills in; the shapes seen so far are `2026-08-31` and +# `2026-08-31 13:53`, plus the occasional non-date. Only the leading date is usable. +EVIDENCE_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}") +JSON_FENCE = re.compile(r"```(?:json)?\s*\n(.*?)```", re.DOTALL) + +GOAL = """\ +You are auditing your own past behaviour. Below is a distilled log of {count} of your earlier +sessions: user and assistant messages in full, tool calls collapsed to `name(args) -> ok|ERROR`. + +Find **recurring mistakes worth fixing** — for example a tool retried with identical arguments +after an error instead of being diagnosed, a runaway loop, a tool chosen where a better one +existed, a multi-step request answered without doing the work, or an instruction you clearly +missed. Judge severity by wasted turns and by the harm to the user's result. + +Rules: +- Report a repeated pattern, with concrete evidence: which sessions, which turns. A single + occurrence is worth reporting only when it matches a pattern in "Known patterns" below — reuse + its exact id and give the honest count for this slice (1 is fine), because those counts add up + across slices. A novel slip you have seen only once here, skip. +- These {count} sessions are one slice of a longer history, not all of it. Count only what you can + see in this slice; the store sums the counts across slices for you. +- Do NOT report a pattern already listed in "Known patterns" below unless you see new occurrences; + reuse its exact `pattern` identifier when you do. +- `pattern` is a short stable kebab-case id (e.g. `retry-without-diagnosis`). Reuse existing ids + from the list rather than inventing a new name for the same thing. +- Include a `patch` only when you are confident: `old_text` must be copied character-for-character + from the current file and must occur exactly once in it. When unsure, give `proposal` alone. +- **Read at most 2 files**, and only to copy `old_text` for a patch. Every extra read costs another + full pass over this whole digest, so spend those two reads on a patch you are sure about. +- At most {max_findings} findings. Fewer, well-evidenced findings are better than a long list. +- **Use no quotation marks of any kind inside JSON string values.** Quote a log line by + writing it plainly, without wrapping it in quotes. A stray `"` breaks the whole answer. + +**Write nothing.** Do not edit, create or delete any file, and do not run any command that +changes state. This is an analysis-only run; a human reviews and applies your suggestions later. + +Answer with exactly one ```json code block and nothing else — no narration before or after: + +```json +{{"findings": [{{"pattern": "retry-without-diagnosis", "severity": "medium", + "diagnosis": "After an HTTP error the same call is repeated with identical arguments…", + "evidence": [{{"session": "websocket_e5a6aa…", "when": "2026-07-11", "excerpt": "web_fetch → ERROR 403 ×4"}}], + "occurrences": 7, "sessions_affected": 4, + "proposal": "Add a hard STOP gate to the Fetching section", + "patch": {{"file": "skills/flight-search/SKILL.md", "old_text": "…", "new_text": "…"}}}}]}} +``` + +If you find nothing worth reporting, answer with an empty `findings` list. + +## Known patterns + +{known_patterns} + +## Sessions + +{digest} +""" + +RETRY_PROMPT = """\ +Your findings were rejected by the validator: + +{errors} + +Answer again with exactly one corrected ```json code block and nothing else. +""" + + +class ReflectError(Exception): + """A failure worth reporting to the user in one line.""" + + +class FindingsError(ReflectError): + """The agent's answer is malformed.""" + + +@dataclass(frozen=True) +class Finding: + """One diagnosed pattern, as filed in reflect/findings.jsonl.""" + + id: str + status: str + created: str + last_seen: str + pattern: str + severity: str + diagnosis: str + evidence: tuple[dict[str, str], ...] + occurrences: int + sessions_affected: int + proposal: str + patch: dict[str, str] | None = None + regression_of: str | None = None + history: tuple[str, ...] = field(default_factory=tuple) + + def to_json(self) -> dict[str, Any]: + record = { + "id": self.id, + "status": self.status, + "created": self.created, + "last_seen": self.last_seen, + "pattern": self.pattern, + "severity": self.severity, + "diagnosis": self.diagnosis, + "evidence": list(self.evidence), + "occurrences": self.occurrences, + "sessions_affected": self.sessions_affected, + "proposal": self.proposal, + } + if self.patch: + record["patch"] = self.patch + if self.regression_of: + record["regression_of"] = self.regression_of + if self.history: + record["history"] = list(self.history) + return record + + +def _config() -> dict[str, Any]: + return json.loads(CONFIG.read_text(encoding="utf-8")) + + +def _workspace(config: dict[str, Any]) -> Path: + configured = config.get("agents", {}).get("defaults", {}).get("workspace") + return Path(configured).expanduser() if configured else WORKSPACE_FALLBACK + + +def _telegram_config(config: dict[str, Any]) -> tuple[str, str]: + telegram = config["channels"]["telegram"] + allow_from = telegram.get("allowFrom") or [] + chat_id = str(allow_from[0]) if allow_from else FALLBACK_CHAT_ID + return telegram["token"], chat_id + + +def _send_telegram(text: str, token: str, chat_id: str) -> None: + url = f"https://api.telegram.org/bot{token}/sendMessage" + payload = urllib.parse.urlencode({"chat_id": chat_id, "text": text}).encode() + request = urllib.request.Request(url, data=payload, method="POST") + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + + +def _git_fingerprint(workspace: Path) -> str: + """HEAD plus the porcelain status — an otherwise unchanged workspace hashes identically. + + This is the guard that the analysis turn stayed read-only. It covers the whole tree, + not just the files the agent was asked about. + """ + try: + head = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=workspace, capture_output=True, text=True, check=True + ).stdout + status = subprocess.run( + ["git", "status", "--porcelain"], cwd=workspace, capture_output=True, text=True, check=True + ).stdout + except (subprocess.CalledProcessError, FileNotFoundError) as error: + raise ReflectError(f"workspace git is unavailable, refusing to run unguarded: {error}") from error + return head + status + + +def _load_state(workspace: Path) -> dict[str, Any]: + path = workspace / STATE_REL + if not path.exists(): + return {"cursor": "", "runs": []} + return json.loads(path.read_text(encoding="utf-8")) + + +def _save_state(workspace: Path, state: dict[str, Any]) -> None: + path = workspace / STATE_REL + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _load_findings(workspace: Path) -> list[dict[str, Any]]: + path = workspace / FINDINGS_REL + if not path.exists(): + return [] + records = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + +def _write_findings(workspace: Path, records: list[dict[str, Any]]) -> None: + path = workspace / FINDINGS_REL + path.parent.mkdir(parents=True, exist_ok=True) + body = "\n".join(json.dumps(record, ensure_ascii=False) for record in records) + path.write_text(body + "\n" if body else "", encoding="utf-8") + + +def _known_patterns(records: list[dict[str, Any]]) -> str: + """Render the pattern vocabulary handed to the model, so it reuses ids instead of renaming.""" + if not records: + return "_(none yet)_" + seen: dict[str, dict[str, Any]] = {} + for record in records: + current = seen.get(record["pattern"]) + if not current or record.get("occurrences", 0) > current.get("occurrences", 0): + seen[record["pattern"]] = record + rows = [] + for pattern, record in sorted(seen.items()): + rows.append(f"- `{pattern}` [{record['status']}] — {record['diagnosis'][:120]}") + return "\n".join(rows) + + +def _new_id(existing: set[str]) -> str: + while True: + candidate = f"f{uuid.uuid4().hex[:4]}" + if candidate not in existing: + return candidate + + +def _text_field(raw: Any, name: str, index: int, limit: int) -> str: + """Trimmed non-empty string, cut to `limit` — an overlong answer is shortened, not discarded.""" + if not isinstance(raw, str) or not raw.strip(): + raise FindingsError(f'- finding {index}: "{name}" must be a non-empty string') + text = raw.strip() + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _positive_int(raw: Any, name: str, index: int) -> int: + if not isinstance(raw, int) or isinstance(raw, bool) or raw < 1: + raise FindingsError(f'- finding {index}: "{name}" must be an integer >= 1') + return raw + + +def _parse_evidence(raw: Any, index: int) -> tuple[dict[str, str], ...]: + if not isinstance(raw, list) or not raw: + raise FindingsError(f'- finding {index}: "evidence" must be a non-empty list') + items = [] + for item in raw: + if not isinstance(item, dict) or not item.get("session"): + raise FindingsError(f'- finding {index}: every evidence item needs a "session"') + items.append({key: str(value) for key, value in item.items() if key in ("session", "when", "excerpt")}) + return tuple(items) + + +def _parse_patch(raw: Any, index: int) -> dict[str, str] | None: + """Validate a proposed patch. Shape only — whether it still applies is checked at review time.""" + if raw is None: + return None + if not isinstance(raw, dict): + raise FindingsError(f'- finding {index}: "patch" must be an object or omitted') + unknown = sorted(str(key) for key in set(raw) - PATCH_KEYS) + if unknown: + raise FindingsError(f"- finding {index}: patch has unknown fields {unknown}") + missing = sorted(PATCH_KEYS - set(raw)) + if missing: + raise FindingsError(f"- finding {index}: patch is missing {missing}") + if not all(isinstance(raw[key], str) for key in PATCH_KEYS): + raise FindingsError(f"- finding {index}: patch fields must be strings") + if not raw["old_text"].strip(): + raise FindingsError(f'- finding {index}: patch "old_text" must not be empty') + if raw["old_text"] == raw["new_text"]: + raise FindingsError(f"- finding {index}: patch changes nothing") + if Path(raw["file"]).is_absolute() or ".." in Path(raw["file"]).parts: + raise FindingsError(f'- finding {index}: patch "file" must be a path inside the workspace') + return {key: raw[key] for key in ("file", "old_text", "new_text")} + + +def _decode_payload(content: str) -> tuple[Any, str]: + """Return (payload, "") or (None, an error the model can act on). + + Naming the offending line, column and surrounding text matters: the usual failure is + an unescaped `"` inside a Czech quotation (`„…"`) in a diagnosis, and a bare + "no parseable json" message gives the retry nothing to fix. + """ + # Fenced blocks first, last one first: that is where the answer is supposed to be, so + # both the successful parse and the reported error come from there rather than from + # the whole reply (whose column numbers would be meaningless to the model). + candidates = [*reversed(JSON_FENCE.findall(content)), content] + best_error = "" + for candidate in candidates: + text = candidate.strip() + if not text: + continue + try: + return json.loads(text), "" + except json.JSONDecodeError as error: + if not best_error: + excerpt = text[max(0, error.pos - 90) : error.pos + 30].replace("\n", " ") + best_error = ( + f"- the JSON is invalid: {error.msg} at line {error.lineno} column {error.colno}\n" + f"- around: …{excerpt}…\n" + '- most likely an unescaped double quote inside a string value; write \\" ' + "or drop the quotes entirely" + ) + return None, best_error or "- the answer contains no parseable ```json block" + + +def _parse_finding(raw: Any, index: int, session_count: int) -> tuple[dict[str, Any], list[str]]: + """Validate one finding, returning it with any notes about what had to be salvaged. + + Raises only when the record itself is unusable. A bad patch costs the patch, not the + finding — the diagnosis and the proposal are still worth putting in front of the user. + """ + if not isinstance(raw, dict): + raise FindingsError(f"- finding {index}: must be a JSON object") + pattern = raw.get("pattern") + if not isinstance(pattern, str) or not PATTERN_RE.match(pattern): + raise FindingsError(f'- finding {index}: "pattern" must be a kebab-case id like `retry-without-diagnosis`') + severity = raw.get("severity") + if severity not in SEVERITIES: + raise FindingsError(f'- finding {index}: "severity" must be one of {list(SEVERITIES)}') + + notes = [] + unknown = sorted(str(key) for key in set(raw) - FINDING_KEYS) + if unknown: + notes.append(f"- finding {index}: ignored unknown fields {unknown}") + try: + patch = _parse_patch(raw.get("patch"), index) + except FindingsError as error: + patch = None + notes.append(f"{error} — kept the finding without it") + + # The counts are the model's own arithmetic and drive both the threshold and the ranking, so + # they get the one check that needs no second opinion: a pattern cannot touch more sessions + # than the times it occurred, nor more than the slice even held. Clamping beats re-asking — + # a whole turn is far too expensive to spend on one wrong integer. + occurrences = _positive_int(raw.get("occurrences"), "occurrences", index) + sessions = _positive_int(raw.get("sessions_affected"), "sessions_affected", index) + ceiling = min(occurrences, session_count) + if sessions > ceiling: + notes.append(f'- finding {index}: "sessions_affected" {sessions} exceeds {ceiling}, clamped') + sessions = ceiling + + finding = { + "pattern": pattern, + "severity": severity, + "diagnosis": _text_field(raw.get("diagnosis"), "diagnosis", index, MAX_DIAGNOSIS_CHARS), + "evidence": _parse_evidence(raw.get("evidence"), index), + "occurrences": occurrences, + "sessions_affected": sessions, + "proposal": _text_field(raw.get("proposal"), "proposal", index, MAX_PROPOSAL_CHARS), + "patch": patch, + } + return finding, notes + + +def parse_findings(content: str, session_count: int) -> tuple[list[dict[str, Any]], list[str]]: + """Parse the agent's answer into raw finding dicts, plus notes on anything dropped. + + Only an unusable *answer* raises: a single malformed finding is discarded and the rest of + the batch survives. Re-asking costs a whole ~420k token turn, far too much to spend on one + bad record — the plan calls for discarding it (plans/reflect-skill.md). + """ + payload, decode_error = _decode_payload(content) + if payload is None: + raise FindingsError(decode_error) + if not isinstance(payload, dict) or not isinstance(payload.get("findings"), list): + raise FindingsError('- the JSON must be an object with a "findings" list') + + raw_findings = payload["findings"] + problems: list[str] = [] + if len(raw_findings) > MAX_FINDINGS_PER_BATCH: + problems.append(f"- kept the first {MAX_FINDINGS_PER_BATCH} of {len(raw_findings)} findings") + raw_findings = raw_findings[:MAX_FINDINGS_PER_BATCH] + + parsed = [] + for index, raw in enumerate(raw_findings, start=1): + try: + finding, notes = _parse_finding(raw, index, session_count) + except FindingsError as error: + problems.append(f"{error} — finding dropped") + continue + parsed.append(finding) + problems += notes + + if raw_findings and not parsed: + raise FindingsError("\n".join(problems)) + return parsed, problems + + +def _fold_evidence(fresh: tuple[dict[str, str], ...], previous: Any) -> tuple[dict[str, str], ...]: + """Newest evidence first, older kept behind it, deduplicated and capped. + + `occurrences` sums across runs while the evidence used to be replaced, so a folded finding + claimed 18 occurrences and showed the two examples from the last batch — a count the user + had no way to audit during review. + """ + seen: set[tuple[str, str]] = set() + kept: list[dict[str, str]] = [] + for item in (*fresh, *(previous or ())): + key = (item.get("session", ""), item.get("excerpt", "")) + if key in seen: + continue + seen.add(key) + kept.append(item) + if len(kept) == MAX_EVIDENCE: + break + return tuple(kept) + + +def _last_seen(evidence: tuple[dict[str, str], ...], created: str) -> str: + """The newest date the evidence actually carries — `created` only says when it was filed.""" + dates = [item["when"][:10] for item in evidence if EVIDENCE_DATE_RE.match(item.get("when", ""))] + return max(dates, default=created) + + +def merge_findings(existing: list[dict[str, Any]], parsed: list[dict[str, Any]], today: str) -> list[Finding]: + """Apply the notification threshold and fold repeats into the record they repeat. + + A pattern seen once stays `watch` and silent; the second sighting promotes it to `open`. + Folding sums the counts, so it also carries the older evidence forward — a cumulative count + with only the newest batch's examples behind it cannot be checked by the person reviewing it. + A pattern that was already applied and comes back is a regression and opens immediately — + but only when the evidence is newer than the fix; evidence from before it describes + behaviour already dealt with, so the finding waits on `watch` until the window catches up. + A pattern the user rejected never opens again — rejection is a decision, not a deferral; + it keeps being counted and stays visible in the report and the pattern list. Rejection is + checked against *every* record of the pattern, not just the newest one: the `watch` record + filed after a rejection would otherwise supersede it and promote the pattern right back. + """ + by_pattern: dict[str, dict[str, Any]] = {} + for record in existing: + current = by_pattern.get(record["pattern"]) + if not current or record["created"] >= current["created"]: + by_pattern[record["pattern"]] = record + used_ids = {record["id"] for record in existing} + rejected_patterns = {record["pattern"] for record in existing if record["status"] == STATUS_REJECTED} + + results = [] + for item in parsed: + previous = by_pattern.get(item["pattern"]) + occurrences = item["occurrences"] + sessions = item["sessions_affected"] + evidence = item["evidence"] + regression_of = None + history: tuple[str, ...] = () + stale_after_fix = False + + if previous and previous["status"] in (STATUS_WATCH, STATUS_OPEN): + occurrences += previous.get("occurrences", 0) + sessions += previous.get("sessions_affected", 0) + evidence = _fold_evidence(item["evidence"], previous.get("evidence")) + history = (*previous.get("history", []), f"{previous['created']}:{previous['id']}") + # A pattern that already came back after a fix stays a regression until it is + # dealt with; folding the repeat in must not quietly drop the flag. + regression_of = previous.get("regression_of") + + last_seen = _last_seen(evidence, today) + if previous and previous["status"] == STATUS_APPLIED: + # Only a sighting *after* the fix is a regression. The window still reaches back over + # sessions that predate it, and flagging those made the fix look undone — worse, it + # opened findings whose whole evidence is about behaviour already dealt with. + if last_seen > previous.get("applied", {}).get("at", "")[:10]: + regression_of = previous["id"] + else: + stale_after_fix = True + + was_rejected = item["pattern"] in rejected_patterns + repeats = occurrences >= 2 and sessions >= 2 + silenced = was_rejected or stale_after_fix + status = STATUS_OPEN if (repeats or regression_of) and not silenced else STATUS_WATCH + + results.append( + Finding( + id=_new_id(used_ids), + status=status, + created=today, + last_seen=last_seen, + pattern=item["pattern"], + severity=item["severity"], + diagnosis=item["diagnosis"], + evidence=evidence, + occurrences=occurrences, + sessions_affected=sessions, + proposal=item["proposal"], + patch=item["patch"], + regression_of=regression_of, + history=history, + ) + ) + used_ids.add(results[-1].id) + return results + + +def supersede(existing: list[dict[str, Any]], merged: list[Finding]) -> list[dict[str, Any]]: + """Drop the watch/open records that the new findings fold in, then append the new ones.""" + folded = {record_id for finding in merged for entry in finding.history for record_id in [entry.split(":")[-1]]} + kept = [record for record in existing if record["id"] not in folded] + return kept + [finding.to_json() for finding in merged] + + +def _rate_line(rate: float, previous: float | None) -> str: + """How often already-known patterns still recur — the one number that says whether this pays off. + + Without a trend the report is only a restatement of findings.jsonl; with it, an applied fix + that did nothing becomes visible in the next run. + """ + trend = f" (previous run {previous:.1f})" if previous is not None else "" + return f"Known patterns: {rate:.1f} occurrences / 100 sessions{trend}." + + +def _window_line(stats: dict[str, Any]) -> str: + """What the findings below actually cover — without it the report reads as if it saw everything.""" + scope = f"from {stats['window_from'][:10]}" if stats.get("window_from") else "from the cursor, no window" + return f"Window: {scope}, batches {stats['batches']}/{stats.get('batches_total', stats['batches'])}." + + +def _seen_line(finding: Finding) -> str: + """First and last sighting: `occurrences` is cumulative, so the count alone hides staleness. + + "Last seen" is the newest date in the evidence, not `created` — a record refiled every night + reported today's date whatever the evidence behind it said. + """ + first = finding.history[0].split(":")[0] if finding.history else finding.created + return f"first seen {first}, last seen {finding.last_seen}" + + +def _is_stale(finding: Finding, stats: dict[str, Any]) -> bool: + """Nothing in the run's window backs this finding any more, so no run will refresh it either.""" + window_from = str(stats.get("window_from") or "")[:10] + return bool(window_from) and finding.last_seen < window_from + + +def render_report(merged: list[Finding], stats: dict[str, Any], previous_rate: float | None, when: datetime) -> str: + """The dated results/ report — the human-readable record behind the Telegram one-liner.""" + lines = [ + f"# Self-reflection {when:%Y-%m-%d}", + "", + f"Analysed {stats['sessions']} sessions in {stats['batches']} batches. Findings: {len(merged)} " + f"({sum(1 for f in merged if f.status == STATUS_OPEN)} to review, " + f"{sum(1 for f in merged if f.status == STATUS_WATCH)} watched).", + "", + _window_line(stats), + _rate_line(stats["repeat_per_100"], previous_rate), + "", + ] + if not merged: + lines.append("Nothing to report.") + return "\n".join(lines) + "\n" + + for finding in sorted(merged, key=lambda f: (f.status != STATUS_OPEN, SEVERITIES.index(f.severity) * -1)): + flag = " — REGRESSION" if finding.regression_of else "" + if _is_stale(finding, stats): + flag += " — STALE" + lines += [ + f"## {finding.id} · `{finding.pattern}` [{finding.status}/{finding.severity}]{flag}", + "", + finding.diagnosis, + "", + f"**Occurrences:** {finding.occurrences}× in {finding.sessions_affected} sessions · {_seen_line(finding)}", + "", + "**Evidence:**", + ] + lines += [ + f"- `{item.get('session', '?')}` {item.get('when', '')} — {item.get('excerpt', '')}".rstrip(" —") + for item in finding.evidence + ] + lines += ["", f"**Proposal:** {finding.proposal}", ""] + if finding.patch: + lines += [ + f"**Patch:** `{finding.patch['file']}`", + "", + "```diff", + *[f"- {line}" for line in finding.patch["old_text"].splitlines()], + *[f"+ {line}" for line in finding.patch["new_text"].splitlines()], + "```", + "", + ] + return "\n".join(lines) + "\n" + + +async def _resolve_findings( + bot: Nanobot, session_key: str, prompt: str, workspace: Path, session_count: int +) -> list[dict[str, Any]]: + """Ask the agent for findings, retrying with validator feedback in the same session. + + A provider failure is counted separately from a malformed answer. The two look alike from + here — nanobot hands back `Error calling LLM: …` as the reply — but blaming the model for + text it never wrote burns a validator attempt and sends it a nonsensical correction. + """ + fingerprint = _git_fingerprint(workspace) + message = prompt + last_error: FindingsError = FindingsError("- no attempt was made") + attempt = 0 + llm_errors = 0 + + while attempt < MAX_ATTEMPTS: + result = await bot.run(message, session_key=session_key) + if result.stop_reason == "error" or result.error: + llm_errors += 1 + detail = (result.error or result.content or "unknown").splitlines()[0] + print(f"[llm-error {llm_errors}/{MAX_LLM_ERROR_RETRIES}] {detail}", file=sys.stderr) + if llm_errors > MAX_LLM_ERROR_RETRIES: + raise ReflectError(f"model unavailable: {detail}") + # The failed turn left an error message and the whole digest in that session; a fresh + # key re-asks the original question instead of paying for the poisoned history. + session_key = f"{session_key}r{llm_errors}" + message = prompt + continue + + attempt += 1 + content = result.content or "" + print(f"[attempt {attempt}/{MAX_ATTEMPTS}] {len(content)} chars", file=sys.stderr) + + if _git_fingerprint(workspace) != fingerprint: + raise ReflectError("the agent modified the workspace during an analysis run — nothing filed") + + try: + parsed, problems = parse_findings(content, session_count) + except FindingsError as error: + last_error = error + print(f"[attempt {attempt}/{MAX_ATTEMPTS}] rejected:\n{error}", file=sys.stderr) + message = RETRY_PROMPT.format(errors=error) + continue + + for problem in problems: + print(f"[attempt {attempt}/{MAX_ATTEMPTS}] {problem}", file=sys.stderr) + return parsed + + raise ReflectError(f"after {MAX_ATTEMPTS} attempts: {str(last_error).splitlines()[0].lstrip('- ')}") + + +def build_prompt(digest: str, known_patterns: str, session_count: int) -> str: + return GOAL.format( + count=session_count, + max_findings=MAX_FINDINGS_PER_BATCH, + known_patterns=known_patterns, + digest=digest, + ) + + +def _open_bot(preset: str): + """Deferred import so a dry run works on a machine without nanobot-ai installed.""" + from nanobot import Nanobot # ty: ignore[unresolved-import] + + return Nanobot.from_config(model_preset=preset) + + +async def _run(workspace: Path, args: argparse.Namespace, now: datetime) -> tuple[str, int]: + state = _load_state(workspace) + # Cursor is the floor, window the ceiling: nothing is analysed twice and nothing older than + # the window is analysed at all, so a run always describes recent behaviour and the backlog + # cannot starve it. Sessions the window skips are skipped for good — the cursor moves past them. + window_from = f"{now - timedelta(days=args.window_days):%Y-%m-%dT%H:%M:%S}" if args.window_days else "" + since = "" if args.all else max(str(state.get("cursor") or ""), window_from) + paths = collect_sessions(workspace / "sessions", since=since) + batches = list(iter_batches(paths, args.budget_chars)) + if args.max_batches: + batches = batches[: args.max_batches] + if not batches: + return "", 0 + + existing = _load_findings(workspace) + known_before = {record["pattern"] for record in existing} + bot = None if args.dry_run else _open_bot(MODEL_PRESET) + runs = state.setdefault("runs", []) + previous_rate = runs[-1].get("repeat_per_100") if runs else None + stats: dict[str, Any] = { + "at": f"{now:%Y-%m-%d %H:%M}", + "window_from": window_from, + "sessions": 0, + "batches": 0, + "batches_total": len(batches), + "open": 0, + "watch": 0, + "repeat_per_100": 0.0, + } + # Keyed by pattern: a pattern found in five batches is one finding, and only the last fold + # of it is the complete one. Appending every batch's copy inflated the report headings, the + # open/watch counts and the Telegram number to several times what the store actually holds. + merged: dict[str, Finding] = {} + session_count = 0 + repeat_occurrences = 0 + deadline_seconds = args.deadline_minutes * 60 + started_at = time.monotonic() + + def commit(batch: list[SessionDigest]) -> None: + """Persist everything analysed so far, after every batch. + + A run that gets cut short — by the deadline, the hard timeout or a dead provider — must + keep the batches it did finish and leave the cursor past them. Doing this once at the end + meant a timeout threw away finished work and left the cursor still pointing at it. + """ + _write_findings(workspace, existing) + stats.update( + sessions=session_count, + batches=stats["batches"] + 1, + open=sum(1 for f in merged.values() if f.status == STATUS_OPEN), + watch=sum(1 for f in merged.values() if f.status == STATUS_WATCH), + repeat_per_100=round(repeat_occurrences / session_count * 100, 1) if session_count else 0.0, + ) + report = workspace / RESULTS_REL / f"{now:%Y-%m-%d}_reflect.md" + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text(render_report(list(merged.values()), stats, previous_rate, now), encoding="utf-8") + # Batches run oldest-first, so the newest start inside one is a safe cursor. Never move it + # backwards: an `--all` run walks the whole backlog and must not undo the nightly progress + # if it dies halfway. + newest = max((item.started for item in batch), default="") + state["cursor"] = max(newest, str(state.get("cursor") or "")) + if stats["batches"] == 1: + runs.append(stats) + _save_state(workspace, state) + + for index, batch in enumerate(batches): + if index and time.monotonic() - started_at > deadline_seconds: + print(f"deadline: stopping after {index}/{len(batches)} batches", file=sys.stderr) + break + + digest = "\n\n".join(item.text for item in batch) + session_count += len(batch) + prompt = build_prompt(digest, _known_patterns(existing), len(batch)) + if args.dry_run: + target = workspace / "tmp" / f"reflect-batch.{index:03d}.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(prompt, encoding="utf-8") + print(f"dry-run: {target} ({len(prompt) / 1000:.0f} kB)", file=sys.stderr) + continue + + session_key = f"reflect:{now:%Y%m%d-%H%M%S}-{index}" + parsed = await _resolve_findings(bot, session_key, prompt, workspace, len(batch)) + # Newly observed occurrences only — the cumulative counts on `merged` carry earlier + # runs with them and would inflate the rate every time a pattern is folded in. + repeat_occurrences += sum(item["occurrences"] for item in parsed if item["pattern"] in known_before) + batch_findings = merge_findings(existing, parsed, f"{now:%Y-%m-%d}") + existing = supersede(existing, batch_findings) + merged.update({finding.pattern: finding for finding in batch_findings}) + commit(batch) + + if args.dry_run: + return "", session_count + + pending = len(batches) - stats["batches"] + opened = [f for f in merged.values() if f.status == STATUS_OPEN] + if not opened: + # Silence is how the starved cursor went unnoticed for three nights: a run that cannot + # keep up finds nothing new, and used to say nothing about the backlog it left behind. + if pending: + scope = window_from[:10] or "the cursor" + return f"🔍 reflect: 0 findings, {pending} batches left (window from {scope}).", session_count + return "", session_count + + patchable = sum(1 for f in opened if f.patch) + regressions = sum(1 for f in opened if f.regression_of) + text = f"🔍 reflect: {len(opened)} findings to review ({patchable} with a patch)" + if regressions: + text += f", {regressions} regressions" + done = stats["batches"] + progress = "" if done == len(batches) else f" Analysed {done}/{len(batches)} batches." + return f"{text}.{progress} Type /reflect.", session_count + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dry-run", action="store_true", help="write prompts to tmp/, call no model") + parser.add_argument("--all", action="store_true", help="ignore the cursor and re-read everything (dry runs only)") + parser.add_argument( + "--window-days", + type=int, + default=DEFAULT_WINDOW_DAYS, + help=f"analyse only sessions from the last N days (default {DEFAULT_WINDOW_DAYS}, 0 = from the cursor)", + ) + parser.add_argument("--budget-chars", type=int, default=DEFAULT_BUDGET_CHARS) + parser.add_argument("--max-batches", type=int, default=0, help="stop after N batches (0 = no limit)") + parser.add_argument( + "--deadline-minutes", + type=int, + default=DEFAULT_DEADLINE_MINUTES, + help="start no new batch past this many minutes (0 = one batch per run)", + ) + parser.add_argument("--workspace", type=Path, help="override the configured workspace (for dry runs)") + args = parser.parse_args(argv) + if args.all and not args.dry_run: + # merge_findings folds by pattern and sums the counts, so re-reading analysed sessions + # inflates every occurrence count. It is a debugging view, not a way to recount. + parser.error("--all re-reads sessions already counted and would inflate the counts; use it with --dry-run") + + now = datetime.now() + # A dry run must work without the server config, so it can be checked from a dev machine. + config = {} if (args.dry_run and args.workspace) else _config() + workspace = args.workspace or _workspace(config) + + try: + message, sessions = asyncio.run(asyncio.wait_for(_run(workspace, args, now), timeout=TIMEOUT_SECONDS)) + failed = False + except TimeoutError: + message = f"🔍 reflect: ERROR — timed out after {TIMEOUT_SECONDS // 60} min, finished batches are saved." + sessions, failed = 0, True + except ReflectError as error: + message, sessions = f"🔍 reflect: ERROR — {error}.", 0 + failed = True + except Exception as error: # noqa: BLE001 — an unattended job must report, not just die + traceback.print_exc() + message, sessions = f"🔍 reflect: ERROR — {type(error).__name__}: {error}.", 0 + failed = True + + if not message: + print(f"reflect_auto: {sessions} sessions, nothing to report", file=sys.stderr) + return 0 + if failed: + print(f"reflect_auto: {message}", file=sys.stderr) + if args.dry_run: + return 1 if failed else 0 + + token, chat_id = _telegram_config(config) + try: + _send_telegram(message, token, chat_id) + except Exception as error: # noqa: BLE001 — delivery failure must not hide the original outcome + print(f"reflect_auto: telegram delivery failed: {error}", file=sys.stderr) + return 1 + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/reflect/scripts/reflect_distill.py b/skills/reflect/scripts/reflect_distill.py new file mode 100644 index 0000000..99f810e --- /dev/null +++ b/skills/reflect/scripts/reflect_distill.py @@ -0,0 +1,338 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""reflect_distill.py — mechanical distillation of session logs for the /reflect skill. + +Turns `sessions/*.jsonl` into a compact, readable timeline the analysing LLM can scan. +Tool results are ~89% of the corpus by volume and carry almost no diagnostic value, so +they collapse to `name(args) -> ok|ERROR, size`; user and assistant prose is kept whole +because that is where intent shows. + +This script makes **no quality judgements** — no error detectors, no ranking, no +"suspicious" flags. Deciding what is a mistake is the LLM's job; mechanical reduction +is this script's job. That split is deliberate (see plans/reflect-skill.md). +""" + +import argparse +import base64 +import binascii +import itertools +import json +import re +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +# workspace/skills/reflect/scripts/reflect_distill.py -> parents[3] = workspace root. +WORKSPACE = Path(__file__).resolve().parents[3] + +# Session-key prefixes that are machinery or throwaway tests, not conversations worth +# diagnosing. `reflect` is here so the skill never analyses its own runs. +NOISE_PREFIXES = ( + "reflect", + "compact-memory-auto", + "detach", + "dream", + "cron", + "cli", + "wiki-compile", + "wiki-capture-test", + "note-compile", + "test", +) + +SESSION_MARKER = "━━━ SESSION" +MIN_MESSAGES = 5 +REASONING_CHARS = 200 +ARG_VALUE_CHARS = 90 + +# The context window is not what bounds a batch — latency is. Measured on real digests this +# corpus runs about 1.2 characters per token, so 500k chars was ~185k tokens per request, whose +# prefill on glm-5.3:cloud kept overrunning the provider timeout and each retry paid for it +# again (2026-09-02 run: 9 timeouts, two turns lost outright). 200k chars ≈ 70k tokens answers +# in one pass; smaller batches also get a more careful read and, since reflect_auto.py files +# findings after every batch, cost nothing but more commit points. It imports this rather than +# keeping its own copy, so a debugging run batches exactly like the real one. +DEFAULT_BUDGET_CHARS = 200_000 + +_WORKSPACE_PATH_RE = re.compile(r"/home/[^/]+/\.nanobot/workspace/") +_WHITESPACE_RE = re.compile(r"\s+") + + +@dataclass(frozen=True) +class SessionDigest: + """One distilled session: identity, size, and the rendered timeline. + + `started` is the raw ISO timestamp exactly as the log holds it, because it doubles as + the cursor `collect_sessions` compares against. Formatting happens only for display. + """ + + name: str + started: str + message_count: int + text: str + + def __len__(self) -> int: + return len(self.text) + + +def _decode_session_name(stem: str) -> str: + """Return the readable session key. + + Older sessions are stored under a base64 filename (`ZHJlYW06...` = `dream:...`); + decoding them is what lets prefix filtering catch that generation too. + """ + if "_" in stem or "-" in stem: + return stem + padded = stem + "=" * (-len(stem) % 4) + try: + decoded = base64.urlsafe_b64decode(padded).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + return stem + return decoded if decoded.isprintable() else stem + + +def is_noise(session_name: str) -> bool: + """True for machinery sessions that carry no diagnostic value. + + Prefix match, not exact: variants like `wiki-capture-test2` or `test-note-search-zzz` + are the same throwaway machinery as their base name. + """ + key = _decode_session_name(session_name) + return key.startswith(NOISE_PREFIXES) + + +def _iter_records(path: Path) -> Iterator[dict]: + """Yield the JSON records of a session log, skipping blank and malformed lines.""" + with path.open(encoding="utf-8", errors="replace") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except json.JSONDecodeError: + continue + + +def _read_records(path: Path) -> list[dict]: + return list(_iter_records(path)) + + +def _format_stamp(raw: str) -> str: + """Display form of the raw ISO timestamp: `2026-07-11 14:02`.""" + return raw[:16].replace("T", " ") if raw else "unknown" + + +def _shorten(text: str, limit: int) -> str: + """Collapse whitespace and cut to `limit`, marking the cut with an ellipsis.""" + flat = _WHITESPACE_RE.sub(" ", text).strip() + return flat if len(flat) <= limit else flat[:limit] + "…" + + +def _shorten_middle(text: str, limit: int) -> str: + """Shorten from the middle, keeping both ends. + + Argument values are usually URLs and paths whose distinguishing part sits at the + *end*; cutting from the front would render two different fetches identical and + hide the difference between a genuine retry and legitimate sequential work. + """ + flat = _WHITESPACE_RE.sub(" ", text).strip() + if len(flat) <= limit: + return flat + head = (limit * 2) // 3 + tail = limit - head + return f"{flat[:head]}…{flat[-tail:]}" + + +def _format_size(char_count: int) -> str: + if char_count < 1000: + return f"{char_count} B" + return f"{char_count / 1000:.1f} kB" + + +def _format_arguments(raw: str) -> str: + """Render tool arguments compactly, stripping the workspace path prefix. + + Falls back to the raw string when arguments are not JSON — some providers emit + partial or malformed argument blobs, and that is itself worth seeing. + """ + try: + parsed = json.loads(raw) if raw else {} + except json.JSONDecodeError: + return _shorten_middle(raw, ARG_VALUE_CHARS) + if not isinstance(parsed, dict): + return _shorten_middle(str(parsed), ARG_VALUE_CHARS) + parts = [] + for key, value in parsed.items(): + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + text = _WORKSPACE_PATH_RE.sub("", text) + parts.append(f"{key}={_shorten_middle(text, ARG_VALUE_CHARS)}") + return ", ".join(parts) + + +def _tool_outcome(content: str) -> str: + """Classify a tool result as ok or ERROR, keeping the error's first line. + + Detection is textual because tool results carry no status field; the leading + `Error:` / `Traceback` / `{"error"` shapes are what the runtime actually emits. + """ + head = content.lstrip()[:80] + if re.match(r'(?i)^(error|exception|traceback|failed|\{"error)', head): + return f"ERROR {_shorten(content, ARG_VALUE_CHARS)}" + return "ok" + + +def distill_session(path: Path) -> SessionDigest | None: + """Render one session file as a timeline, or None when it should be skipped. + + Skipped: machinery sessions (see NOISE_PREFIXES) and sessions shorter than + MIN_MESSAGES, which are too short to show a behavioural pattern. + """ + if is_noise(path.stem): + return None + + session_key = _decode_session_name(path.stem) + records = _read_records(path) + started = "" + lines: list[str] = [] + message_count = 0 + pending_calls: dict[str, str] = {} + + for record in records: + if record.get("_type") == "metadata": + started = str(record.get("created_at") or "") + continue + # Slash commands are handled by the runtime, not by the agent's reasoning. + if record.get("_command"): + continue + + role = record.get("role") + content = record.get("content") or "" + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False) + if not started: + started = str(record.get("timestamp") or "") + + if role == "user": + message_count += 1 + lines.append(f"u: {content.strip()}") + elif role == "assistant": + message_count += 1 + reasoning = record.get("reasoning_content") or "" + if reasoning: + lines.append(f" ~ {_shorten(reasoning, REASONING_CHARS)}") + if content.strip(): + lines.append(f"a: {content.strip()}") + for call in record.get("tool_calls") or []: + function = call.get("function") or {} + name = function.get("name") or "?" + pending_calls[call.get("id") or ""] = name + lines.append(f"a: → {name}({_format_arguments(function.get('arguments') or '')})") + elif role == "tool": + message_count += 1 + name = record.get("name") or pending_calls.get(record.get("tool_call_id") or "", "?") + lines.append(f" ← {name}: {_tool_outcome(content)}, {_format_size(len(content))}") + + if message_count < MIN_MESSAGES: + return None + + # Box-drawing marker, not a markdown heading: assistant prose is full of `###`, + # so a heading would not read as a session boundary. + header = f"{SESSION_MARKER} {session_key} | {_format_stamp(started)} | {message_count} messages" + return SessionDigest( + name=session_key, + started=started, + message_count=message_count, + text="\n".join([header, *lines]), + ) + + +def _session_start(path: Path) -> str: + """First timestamp inside the file — mtime is unreliable after bulk file moves. + + Reads only the opening records: this runs for every session on every run, and parsing + whole files here would mean reading the entire corpus twice. + """ + for record in itertools.islice(_iter_records(path), 5): + stamp = record.get("created_at") or record.get("timestamp") + if stamp: + return str(stamp) + return "" + + +def collect_sessions(sessions_dir: Path, since: str = "") -> list[Path]: + """Session files newer than `since` (ISO timestamp), oldest first.""" + paths = [] + for path in sorted(sessions_dir.glob("*.jsonl")): + if is_noise(path.stem): + continue + start = _session_start(path) + if since and start and start <= since: + continue + paths.append((start, path)) + return [path for _, path in sorted(paths)] + + +def iter_batches(paths: list[Path], budget_chars: int) -> Iterator[list[SessionDigest]]: + """Group distilled sessions into batches that fit the per-turn character budget. + + A single session larger than the budget still gets its own batch — truncating it + would hide exactly the long runaway loops worth finding. + """ + batch: list[SessionDigest] = [] + size = 0 + for path in paths: + digest = distill_session(path) + if digest is None: + continue + if batch and size + len(digest) > budget_chars: + yield batch + batch, size = [], 0 + batch.append(digest) + size += len(digest) + if batch: + yield batch + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--sessions-dir", type=Path, default=WORKSPACE / "sessions") + parser.add_argument("--since", default="", help="ISO timestamp; only sessions started after it") + parser.add_argument("--budget-chars", type=int, default=DEFAULT_BUDGET_CHARS) + parser.add_argument("--out", type=Path, help="write batches to OUT.NNN.md instead of stdout") + parser.add_argument("--stats", action="store_true", help="print a size summary to stderr") + args = parser.parse_args(argv) + + if not args.sessions_dir.is_dir(): + print(f"sessions dir not found: {args.sessions_dir}", file=sys.stderr) + return 1 + + paths = collect_sessions(args.sessions_dir, args.since) + batches = list(iter_batches(paths, args.budget_chars)) + + for index, batch in enumerate(batches): + body = "\n\n".join(digest.text for digest in batch) + if args.out: + target = args.out.with_suffix(f".{index:03d}.md") + target.write_text(body, encoding="utf-8") + else: + print(body) + + if args.stats: + sessions = sum(len(batch) for batch in batches) + chars = sum(len(digest) for batch in batches for digest in batch) + print( + f"{datetime.now():%Y-%m-%d %H:%M} candidates {len(paths)}, " + f"distilled {sessions} sessions, {len(batches)} batches, {chars / 1000:.0f} kB", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/reflect/tests/test_reflect_apply.py b/skills/reflect/tests/test_reflect_apply.py new file mode 100644 index 0000000..e8d4772 --- /dev/null +++ b/skills/reflect/tests/test_reflect_apply.py @@ -0,0 +1,381 @@ +"""Tests for reflect_apply.py — the gate that stands between a finding and a real edit. + +These are the guarantees the user asked for: nothing is applied without an explicit, +per-finding approval, an ambiguous patch is refused rather than guessed at, and every +applied change is one revertable commit touching one file. +""" + +import json +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import reflect_apply + +NOW = datetime(2026, 9, 8, 10, 30) +TARGET_REL = "skills/demo/SKILL.md" +ORIGINAL = "# Demo\n\nIf the fetch fails, try again.\n\nDone.\n" + + +def _record(**overrides): + record = { + "id": "f7a2", + "status": "open", + "created": "2026-09-08", + "pattern": "retry-without-diagnosis", + "severity": "medium", + "diagnosis": "Repeats a call without diagnosing it.", + "evidence": [{"session": "websocket_abc"}], + "occurrences": 7, + "sessions_affected": 4, + "proposal": "Add a hard STOP gate.", + "patch": { + "file": TARGET_REL, + "old_text": "If the fetch fails, try again.", + "new_text": "If the fetch fails, STOP and diagnose.", + }, + } + record.update(overrides) + return record + + +@pytest.fixture +def workspace(tmp_path): + """A miniature workspace that is a real git repo, like the server's.""" + target = tmp_path / TARGET_REL + target.parent.mkdir(parents=True) + target.write_text(ORIGINAL, encoding="utf-8") + (tmp_path / "reflect").mkdir() + (tmp_path / "reflect" / "findings.jsonl").write_text(json.dumps(_record()) + "\n", encoding="utf-8") + + for args in ( + ["init", "-q"], + ["config", "user.email", "test@example.com"], + ["config", "user.name", "test"], + ["add", "-A"], + ["commit", "-q", "-m", "init"], + ): + subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True) + return tmp_path + + +def _write_findings(workspace, *records): + path = workspace / "reflect" / "findings.jsonl" + path.write_text("\n".join(json.dumps(r) for r in records) + "\n", encoding="utf-8") + + +def _findings(workspace): + lines = (workspace / "reflect" / "findings.jsonl").read_text(encoding="utf-8").splitlines() + return [json.loads(line) for line in lines if line.strip()] + + +def _git(workspace, *args): + return subprocess.run(["git", *args], cwd=workspace, capture_output=True, text=True, check=True).stdout.strip() + + +class TestRefusals: + def test_missing_original_text_is_refused(self, workspace): + _write_findings(workspace, _record(patch={**_record()["patch"], "old_text": "text that is not there"})) + with pytest.raises(reflect_apply.ApplyError, match="no longer in"): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL + + def test_ambiguous_original_text_is_refused(self, workspace): + """Two matches means the intended location is a guess — refuse, never guess.""" + (workspace / TARGET_REL).write_text(ORIGINAL + "If the fetch fails, try again.\n", encoding="utf-8") + before = (workspace / TARGET_REL).read_text(encoding="utf-8") + with pytest.raises(reflect_apply.ApplyError, match="occurs 2"): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == before + + def test_a_finding_without_a_patch_is_refused(self, workspace): + record = _record() + del record["patch"] + _write_findings(workspace, record) + with pytest.raises(reflect_apply.ApplyError, match="no patch"): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + + @pytest.mark.parametrize("status", ["watch", "applied", "rejected"]) + def test_only_open_findings_can_be_applied(self, workspace, status): + """`watch` findings were never shown to the user, so they were never approved.""" + _write_findings(workspace, _record(status=status)) + with pytest.raises(reflect_apply.ApplyError): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + + def test_unknown_id_is_refused(self, workspace): + with pytest.raises(reflect_apply.ApplyError, match="no finding"): + reflect_apply.apply_finding(workspace, "nope", None, NOW) + + def test_path_outside_the_workspace_is_refused(self, workspace): + _write_findings(workspace, _record(patch={**_record()["patch"], "file": "../../../etc/passwd"})) + with pytest.raises(reflect_apply.ApplyError): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + + def test_refusal_leaves_the_store_untouched(self, workspace): + _write_findings(workspace, _record(patch={**_record()["patch"], "old_text": "absent"})) + with pytest.raises(reflect_apply.ApplyError): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert _findings(workspace)[0]["status"] == "open" + + +class TestApply: + def test_patch_is_applied_and_committed(self, workspace): + message = reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert "STOP and diagnose" in (workspace / TARGET_REL).read_text(encoding="utf-8") + assert "commit" in message + assert "retry-without-diagnosis (f7a2)" in _git(workspace, "log", "-1", "--pretty=%s") + + def test_commit_contains_only_the_patched_file(self, workspace): + """Dream and other skills leave unrelated work in progress — it must not be swept in.""" + (workspace / "unrelated.md").write_text("someone else's work\n", encoding="utf-8") + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert _git(workspace, "show", "--name-only", "--pretty=", "HEAD") == TARGET_REL + assert "unrelated.md" in _git(workspace, "status", "--porcelain") + + def test_prior_edits_to_the_same_file_are_checkpointed_first(self, workspace): + """The patch commit must be the patch alone, so the revert is exact.""" + target = workspace / TARGET_REL + target.write_text(ORIGINAL + "\nhand-written note\n", encoding="utf-8") + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + subjects = _git(workspace, "log", "-2", "--pretty=%s").splitlines() + assert subjects[1] == "reflect: checkpoint before f7a2" + assert "hand-written note" in target.read_text(encoding="utf-8") + + def test_revert_restores_the_original(self, workspace): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + sha = _findings(workspace)[0]["applied"]["sha"] + subprocess.run(["git", "revert", "--no-edit", sha], cwd=workspace, check=True, capture_output=True) + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL + + def test_store_records_status_sha_and_file(self, workspace): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + record = _findings(workspace)[0] + assert record["status"] == "applied" + assert record["applied"]["file"] == TARGET_REL + assert record["applied"]["sha"] + + def test_other_records_survive_byte_identical(self, workspace): + other = _record(id="f0002", pattern="other-thing", status="watch") + _write_findings(workspace, _record(), other) + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert _findings(workspace)[1] == other + + def test_audit_line_is_appended(self, workspace): + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + log = (workspace / "log" / "reflect.log").read_text(encoding="utf-8") + assert "APPLIED f7a2 [retry-without-diagnosis]" in log + + def test_user_edited_replacement_is_used_and_recorded(self, workspace): + reflect_apply.apply_finding(workspace, "f7a2", "STOP. Read the status code first.", NOW) + assert "Read the status code first." in (workspace / TARGET_REL).read_text(encoding="utf-8") + assert _findings(workspace)[0]["applied"]["edited_by_user"] is True + + +class TestCheckAndReject: + def test_check_reports_success_without_touching_anything(self, workspace): + exit_code = reflect_apply.main(["--id", "f7a2", "--check", "--workspace", str(workspace)]) + assert exit_code == 0 + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL + assert _findings(workspace)[0]["status"] == "open" + + def test_check_prints_the_diff_the_skill_shows(self, workspace, capsys): + """The skill presents whatever this prints, so the diff has to come from here.""" + reflect_apply.main(["--id", "f7a2", "--check", "--workspace", str(workspace)]) + printed = capsys.readouterr().out + assert "-If the fetch fails, try again." in printed + assert "+If the fetch fails, STOP and diagnose." in printed + + def test_check_fails_on_a_stale_patch(self, workspace): + _write_findings(workspace, _record(patch={**_record()["patch"], "old_text": "absent"})) + assert reflect_apply.main(["--id", "f7a2", "--check", "--workspace", str(workspace)]) == 2 + + def test_failed_commit_restores_the_file(self, workspace, monkeypatch, capsys): + """Exit code 2 tells the skill nothing happened — so nothing may be left behind.""" + real_git = reflect_apply._git + + def failing_git(ws, *args): + if args[0] == "commit" and "checkpoint" not in " ".join(args): + raise subprocess.CalledProcessError(1, ["git", *args], stderr="empty ident name") + return real_git(ws, *args) + + monkeypatch.setattr(reflect_apply, "_git", failing_git) + exit_code = reflect_apply.main(["--id", "f7a2", "--workspace", str(workspace)]) + + assert exit_code == 2 + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL + assert _findings(workspace)[0]["status"] == "open" + assert _git(workspace, "status", "--porcelain") == "" + + def test_reject_closes_the_finding_without_editing(self, workspace): + reflect_apply.reject_finding(workspace, "f7a2", "false positive", NOW) + assert _findings(workspace)[0]["status"] == "rejected" + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL + assert "REJECTED f7a2" in (workspace / "log" / "reflect.log").read_text(encoding="utf-8") + + def test_cli_refusal_exits_nonzero(self, workspace): + assert reflect_apply.main(["--id", "unknown", "--workspace", str(workspace)]) == 2 + + +class TestSetPatch: + """A patch drafted during the review is filed by the script — the agent never touches the store. + + Before this existed the agent had to edit findings.jsonl with an ad-hoc script to get a patch + in, which put an LLM inside the audit trail and left unapplicable patches behind on a miss. + """ + + def _patch_file(self, tmp_path, **overrides): + patch = {"file": TARGET_REL, "old_text": "Done.", "new_text": "Done, and diagnosed."} + patch.update(overrides) + path = tmp_path / "patch.json" + path.write_text(json.dumps(patch), encoding="utf-8") + return path + + def _set(self, workspace, path, finding_id="f7a2"): + return reflect_apply.main(["--id", finding_id, "--set-patch", str(path), "--workspace", str(workspace)]) + + def _without_patch(self, workspace): + record = _record() + del record["patch"] + _write_findings(workspace, record) + return record + + def test_a_patch_that_does_not_apply_never_reaches_the_store(self, workspace, tmp_path): + """Verification runs before the write, so a failed attempt leaves nothing behind.""" + self._without_patch(workspace) + assert self._set(workspace, self._patch_file(tmp_path, old_text="TEXT THAT IS NOT THERE")) == 2 + assert "patch" not in _findings(workspace)[0] + + def test_an_ambiguous_patch_never_reaches_the_store(self, workspace, tmp_path): + (workspace / TARGET_REL).write_text(ORIGINAL + "Done.\n", encoding="utf-8") + self._without_patch(workspace) + assert self._set(workspace, self._patch_file(tmp_path)) == 2 + assert "patch" not in _findings(workspace)[0] + + def test_a_valid_patch_is_filed_with_its_provenance(self, workspace, tmp_path): + self._without_patch(workspace) + assert self._set(workspace, self._patch_file(tmp_path)) == 0 + + record = _findings(workspace)[0] + assert record["patch"]["new_text"] == "Done, and diagnosed." + assert record["patch_drafted_at"], "drafted during the review, not proposed by the analysis" + assert record["status"] == "open", "drafting a patch decides nothing" + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL, "nothing is applied yet" + assert "DRAFTED f7a2 [retry-without-diagnosis]" in (workspace / "log" / "reflect.log").read_text("utf-8") + + def test_the_diff_comes_back_in_the_same_call(self, workspace, tmp_path, capsys): + """Two calls instead of six: draft and show. Asking --check afterwards would be a third.""" + self._without_patch(workspace) + self._set(workspace, self._patch_file(tmp_path)) + printed = capsys.readouterr().out + assert "-Done." in printed + assert "+Done, and diagnosed." in printed + + def test_a_wrong_first_draft_can_be_replaced(self, workspace, tmp_path): + """The guard is the `open` status, not the absence of a patch — first drafts get it wrong.""" + assert self._set(workspace, self._patch_file(tmp_path)) == 0 + assert _findings(workspace)[0]["patch"]["old_text"] == "Done." + + @pytest.mark.parametrize("status", ["watch", "applied", "rejected"]) + def test_only_open_findings_can_be_drafted_for(self, workspace, tmp_path, status): + _write_findings(workspace, _record(status=status)) + assert self._set(workspace, self._patch_file(tmp_path)) == 2 + + def test_a_patch_outside_the_workspace_is_refused(self, workspace, tmp_path): + self._without_patch(workspace) + assert self._set(workspace, self._patch_file(tmp_path, file="../../../etc/passwd")) == 2 + assert "patch" not in _findings(workspace)[0] + + @pytest.mark.parametrize("body", ['{"file": "a.md"}', '{"file": 1, "old_text": "a", "new_text": "b"}', "not json"]) + def test_a_malformed_patch_file_is_refused(self, workspace, tmp_path, body): + self._without_patch(workspace) + path = tmp_path / "patch.json" + path.write_text(body, encoding="utf-8") + assert self._set(workspace, path) == 2 + assert "patch" not in _findings(workspace)[0] + + @pytest.mark.parametrize("other", [["--check"], ["--skip"], ["--reject", "--reason", "no"]]) + def test_set_patch_excludes_the_other_decisions(self, workspace, tmp_path, other): + path = self._patch_file(tmp_path) + with pytest.raises(SystemExit): + reflect_apply.main(["--id", "f7a2", "--set-patch", str(path), *other, "--workspace", str(workspace)]) + + def test_a_drafted_patch_can_then_be_applied(self, workspace, tmp_path): + self._without_patch(workspace) + self._set(workspace, self._patch_file(tmp_path)) + reflect_apply.apply_finding(workspace, "f7a2", None, NOW) + assert "Done, and diagnosed." in (workspace / TARGET_REL).read_text(encoding="utf-8") + + +class TestDecisionRecord: + """A decision the audit cannot reconstruct is not recorded — the reason and the skips too.""" + + def _log(self, workspace) -> str: + return (workspace / "log" / "reflect.log").read_text(encoding="utf-8") + + def test_reject_needs_a_reason(self, workspace): + with pytest.raises(SystemExit): + reflect_apply.main(["--id", "f7a2", "--reject", "--workspace", str(workspace)]) + assert _findings(workspace)[0]["status"] == "open" + + def test_a_blank_reason_does_not_count(self, workspace): + with pytest.raises(SystemExit): + reflect_apply.main(["--id", "f7a2", "--reject", "--reason", " ", "--workspace", str(workspace)]) + assert _findings(workspace)[0]["status"] == "open" + + def test_a_reason_without_reject_is_refused(self, workspace): + with pytest.raises(SystemExit): + reflect_apply.main(["--id", "f7a2", "--reason", "because", "--workspace", str(workspace)]) + + def test_the_reason_lands_in_the_record_and_the_log(self, workspace): + exit_code = reflect_apply.main( + ["--id", "f7a2", "--reject", "--reason", "false positive", "--workspace", str(workspace)] + ) + assert exit_code == 0 + rejected = _findings(workspace)[0]["rejected"] + assert rejected["reason"] == "false positive" + assert rejected["at"] + assert "false positive" in self._log(workspace) + + def test_skip_counts_up_and_decides_nothing(self, workspace): + for expected in (1, 2, 3): + assert reflect_apply.main(["--id", "f7a2", "--skip", "--workspace", str(workspace)]) == 0 + record = _findings(workspace)[0] + assert record["skipped"]["count"] == expected + assert record["status"] == "open", "a skip is a deferral, not a decision" + + assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL + assert "SKIPPED f7a2 [retry-without-diagnosis] ×3" in self._log(workspace) + + def test_an_edited_patch_keeps_the_model_proposal(self, workspace, tmp_path): + """Overwriting `patch` lost the only record of what was proposed versus approved.""" + replacement = tmp_path / "new.txt" + replacement.write_text("If the fetch fails, ask the user.", encoding="utf-8") + exit_code = reflect_apply.main( + ["--id", "f7a2", "--new-text-file", str(replacement), "--workspace", str(workspace)] + ) + + assert exit_code == 0 + record = _findings(workspace)[0] + assert record["patch"]["new_text"] == "If the fetch fails, STOP and diagnose.", "the model's proposal" + assert record["applied"]["new_text"] == "If the fetch fails, ask the user.", "what the user approved" + assert record["applied"]["edited_by_user"] is True + assert "ask the user" in (workspace / TARGET_REL).read_text(encoding="utf-8") + assert "APPLIED-EDITED f7a2" in self._log(workspace) + + def test_a_pre_migration_record_still_works(self, workspace): + """Findings decided before the reason existed carry a flat `rejected_at` — leave them be.""" + _write_findings(workspace, _record(id="fold", status="rejected", rejected_at="2026-09-01 10:55"), _record()) + assert reflect_apply.main(["--id", "f7a2", "--skip", "--workspace", str(workspace)]) == 0 + assert _findings(workspace)[0]["rejected_at"] == "2026-09-01 10:55" + + def test_skip_refuses_a_finding_that_is_already_decided(self, workspace): + _write_findings(workspace, _record(status="applied")) + assert reflect_apply.main(["--id", "f7a2", "--skip", "--workspace", str(workspace)]) == 2 + + def test_skip_and_reject_are_mutually_exclusive(self, workspace): + with pytest.raises(SystemExit): + reflect_apply.main(["--id", "f7a2", "--skip", "--reject", "--reason", "x", "--workspace", str(workspace)]) diff --git a/skills/reflect/tests/test_reflect_auto.py b/skills/reflect/tests/test_reflect_auto.py new file mode 100644 index 0000000..512290b --- /dev/null +++ b/skills/reflect/tests/test_reflect_auto.py @@ -0,0 +1,701 @@ +"""Tests for reflect_auto.py — answer validation, batch persistence and the notification threshold.""" + +import asyncio +import json +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import reflect_auto + +TODAY = "2026-09-08" +WHEN = datetime(2026, 9, 8) + + +def _stats(**overrides): + """The per-run statistics dict that feeds both the report and state.json.""" + stats = {"at": "2026-09-08 03:30", "sessions": 4, "batches": 1, "open": 0, "watch": 0, "repeat_per_100": 0.0} + stats.update(overrides) + return stats + + +def _raw_finding(**overrides): + finding = { + "pattern": "retry-without-diagnosis", + "severity": "medium", + "diagnosis": "After an error the same call is repeated with identical arguments.", + "evidence": [{"session": "websocket_abc", "when": "2026-07-11", "excerpt": "web_fetch → ERROR ×4"}], + "occurrences": 1, + "sessions_affected": 1, + "proposal": "Add a hard STOP gate.", + } + finding.update(overrides) + return finding + + +def _answer(*findings) -> str: + return "```json\n" + json.dumps({"findings": list(findings)}) + "\n```" + + +def _parsed(answer: str, session_count: int = 10) -> list[dict]: + """Only the findings — notes about what was salvaged are asserted where they matter.""" + return reflect_auto.parse_findings(answer, session_count)[0] + + +def _filed(**overrides): + """A record as it would already sit in findings.jsonl.""" + record = { + "id": "f0001", + "status": "watch", + "created": "2026-09-01", + "pattern": "retry-without-diagnosis", + "severity": "medium", + "diagnosis": "After an error the same call is repeated.", + "evidence": [{"session": "websocket_old"}], + "occurrences": 1, + "sessions_affected": 1, + "proposal": "Add a hard STOP gate.", + } + record.update(overrides) + return record + + +class TestParseFindings: + def test_accepts_a_well_formed_answer(self): + parsed = _parsed(_answer(_raw_finding())) + assert parsed[0]["pattern"] == "retry-without-diagnosis" + assert parsed[0]["patch"] is None + + def test_accepts_an_empty_findings_list(self): + assert _parsed(_answer()) == [] + + def test_accepts_bare_json_without_a_fence(self): + assert _parsed(json.dumps({"findings": []})) == [] + + def test_ignores_narration_around_the_block(self): + answer = "Here are the findings:\n" + _answer(_raw_finding()) + "\nDone." + assert len(_parsed(answer)) == 1 + + def test_rejects_an_answer_without_json(self): + with pytest.raises(reflect_auto.FindingsError): + reflect_auto.parse_findings("I found three problems but I am not sending JSON.", 10) + + def test_unescaped_quote_error_points_at_the_offending_text(self): + """The real failure seen on the first live run: a Czech „…" closing with ASCII ". + + A bare "no parseable json" message gives the retry nothing to work with, so the + error must name the line, the column and the surrounding text. + """ + broken = '```json\n{"findings": [{"pattern": "x", "diagnosis": "runtime said („blocked") and then"}]}\n```' + with pytest.raises(reflect_auto.FindingsError) as raised: + reflect_auto.parse_findings(broken, 10) + message = str(raised.value) + assert "line 1 column" in message + assert "blocked" in message + assert "unescaped double quote" in message + + def test_error_is_located_inside_the_block_not_the_whole_reply(self): + """Column numbers measured across the fence would be meaningless to the model.""" + broken = 'Here is the result:\n\n```json\n{"findings": [{"diagnosis": "a "b" c"}]}\n```' + with pytest.raises(reflect_auto.FindingsError) as raised: + reflect_auto.parse_findings(broken, 10) + assert "line 1 column" in str(raised.value) + + def test_last_block_wins_when_the_model_shows_its_work(self): + first = json.dumps({"findings": [_raw_finding(pattern="draft-version")]}) + second = json.dumps({"findings": [_raw_finding(pattern="final-version")]}) + answer = f"First draft:\n```json\n{first}\n```\nCorrected:\n```json\n{second}\n```" + assert _parsed(answer)[0]["pattern"] == "final-version" + + @pytest.mark.parametrize( + "override", + [ + {"pattern": "Retry Without Diagnosis"}, + {"pattern": "ab"}, + {"severity": "critical"}, + {"diagnosis": ""}, + {"occurrences": 0}, + {"occurrences": "sedm"}, + {"evidence": []}, + {"evidence": [{"when": "2026-07-11"}]}, + ], + ) + def test_malformed_finding_is_dropped_and_the_others_survive(self, override): + """Re-asking costs a ~420k token turn — one bad record must not throw the batch away.""" + parsed, problems = reflect_auto.parse_findings( + _answer(_raw_finding(**override), _raw_finding(pattern="something-else")), 10 + ) + assert [finding["pattern"] for finding in parsed] == ["something-else"] + assert any("dropped" in problem for problem in problems) + + def test_an_answer_with_nothing_usable_still_raises(self): + """Nothing salvageable means the turn was wasted — that is worth one retry.""" + with pytest.raises(reflect_auto.FindingsError): + _parsed(_answer(_raw_finding(severity="critical"))) + + def test_unknown_fields_are_ignored_not_fatal(self): + parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(confidence=0.9)), 10) + assert "confidence" not in parsed[0] + assert any("confidence" in problem for problem in problems) + + def test_extra_findings_are_trimmed_not_the_batch(self): + many = [_raw_finding(pattern=f"pattern-{index}") for index in range(reflect_auto.MAX_FINDINGS_PER_BATCH + 1)] + parsed, problems = reflect_auto.parse_findings(_answer(*many), 10) + assert len(parsed) == reflect_auto.MAX_FINDINGS_PER_BATCH + assert any("kept the first" in problem for problem in problems) + + def test_overlong_diagnosis_is_truncated(self): + parsed = _parsed(_answer(_raw_finding(diagnosis="x" * (reflect_auto.MAX_DIAGNOSIS_CHARS + 50)))) + assert len(parsed[0]["diagnosis"]) == reflect_auto.MAX_DIAGNOSIS_CHARS + assert parsed[0]["diagnosis"].endswith("…") + + +class TestParsePatch: + def _patch(self, **overrides): + patch = {"file": "skills/note/SKILL.md", "old_text": "try again", "new_text": "STOP and diagnose"} + patch.update(overrides) + return patch + + def test_accepts_a_complete_patch(self): + parsed = _parsed(_answer(_raw_finding(patch=self._patch()))) + assert parsed[0]["patch"]["file"] == "skills/note/SKILL.md" + + @pytest.mark.parametrize( + "override", + [ + {"old_text": ""}, + {"new_text": "try again"}, + {"file": "/etc/passwd"}, + {"file": "../../../etc/passwd"}, + ], + ) + def test_unsafe_or_empty_patch_is_dropped_but_the_finding_survives(self, override): + """The diagnosis and the proposal are still worth reviewing without a patch.""" + parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(patch=self._patch(**override))), 10) + assert len(parsed) == 1 + assert parsed[0]["patch"] is None + assert any("kept the finding without it" in problem for problem in problems) + + def test_incomplete_patch_is_dropped_but_the_finding_survives(self): + parsed = _parsed(_answer(_raw_finding(patch={"file": "a.md", "old_text": "x"}))) + assert parsed[0]["patch"] is None + + +class TestCounts: + """The counts drive the threshold and the ranking, and nothing but this checks them.""" + + def test_more_sessions_than_occurrences_is_impossible(self): + """Seen live: a filed finding claimed 4 occurrences across 5 sessions.""" + parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(occurrences=4, sessions_affected=5)), 10) + assert parsed[0]["sessions_affected"] == 4 + assert any("clamped" in problem for problem in problems) + + def test_more_sessions_than_the_slice_held_is_impossible(self): + parsed, _ = reflect_auto.parse_findings(_answer(_raw_finding(occurrences=9, sessions_affected=6)), 3) + assert parsed[0]["sessions_affected"] == 3 + + def test_a_coherent_count_is_left_alone(self): + parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(occurrences=7, sessions_affected=4)), 10) + assert (parsed[0]["occurrences"], parsed[0]["sessions_affected"]) == (7, 4) + assert problems == [] + + +class TestEvidenceFolding: + """`occurrences` sums across runs, so the examples behind it have to survive the fold.""" + + def _evidence(self, session: str) -> list[dict]: + return [{"session": session, "when": "2026-09-02", "excerpt": "web_fetch → ERROR"}] + + def test_the_fold_keeps_the_older_evidence(self): + existing = [_filed(evidence=self._evidence("websocket_old"))] + raw = _raw_finding(evidence=self._evidence("websocket_new")) + merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY) + + sessions = [item["session"] for item in merged[0].evidence] + assert sessions == ["websocket_new", "websocket_old"], "newest first, older behind it" + + def test_the_same_evidence_twice_is_kept_once(self): + existing = [_filed(evidence=self._evidence("websocket_same"))] + raw = _raw_finding(evidence=self._evidence("websocket_same")) + merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY) + + assert len(merged[0].evidence) == 1 + + def test_the_evidence_list_is_capped(self): + older = [{"session": f"websocket_{i}", "excerpt": str(i)} for i in range(10)] + raw = _raw_finding(evidence=self._evidence("websocket_new")) + merged = reflect_auto.merge_findings([_filed(evidence=older)], _parsed(_answer(raw)), TODAY) + + assert len(merged[0].evidence) == reflect_auto.MAX_EVIDENCE + + def test_a_regression_does_not_mix_evidence_from_before_the_fix(self): + existing = [_filed(status="applied", evidence=self._evidence("websocket_old"))] + raw = _raw_finding(evidence=self._evidence("websocket_new")) + merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY) + + assert [item["session"] for item in merged[0].evidence] == ["websocket_new"] + + +class TestThreshold: + def test_first_sighting_stays_silent(self): + """A single occurrence is noise, not a pattern — it must not reach Telegram.""" + merged = reflect_auto.merge_findings([], _parsed(_answer(_raw_finding())), TODAY) + assert merged[0].status == reflect_auto.STATUS_WATCH + + def test_repeat_within_one_batch_opens_immediately(self): + raw = _raw_finding(occurrences=7, sessions_affected=4) + merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY) + assert merged[0].status == reflect_auto.STATUS_OPEN + + def test_many_occurrences_in_one_session_stay_silent(self): + """One session looping seven times is still one session — not yet a habit.""" + raw = _raw_finding(occurrences=7, sessions_affected=1) + merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY) + assert merged[0].status == reflect_auto.STATUS_WATCH + + def test_second_sighting_promotes_a_watched_pattern(self): + merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding())), TODAY) + assert merged[0].status == reflect_auto.STATUS_OPEN + assert merged[0].occurrences == 2 + assert merged[0].sessions_affected == 2 + + def test_promotion_records_the_superseded_record(self): + merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding())), TODAY) + assert merged[0].history == ("2026-09-01:f0001",) + + def test_reappearing_after_apply_is_a_regression(self): + """A fixed pattern coming back must open at once, however few the occurrences.""" + merged = reflect_auto.merge_findings([_filed(status="applied")], _parsed(_answer(_raw_finding())), TODAY) + assert merged[0].status == reflect_auto.STATUS_OPEN + assert merged[0].regression_of == "f0001" + assert merged[0].occurrences == 1 + + def test_evidence_from_before_the_fix_is_not_a_regression(self): + """The window reaches back over sessions that predate the fix; they say nothing about it.""" + applied = _filed(status="applied", applied={"at": "2026-09-01 10:00", "sha": "abc", "file": "SOUL.md"}) + raw = _raw_finding(occurrences=9, sessions_affected=5, evidence=[{"session": "ws", "when": "2026-08-31"}]) + merged = reflect_auto.merge_findings([applied], _parsed(_answer(raw)), TODAY) + + assert merged[0].regression_of is None + assert merged[0].status == reflect_auto.STATUS_WATCH, "a finding about the past must not open" + + def test_evidence_from_after_the_fix_is_a_regression(self): + applied = _filed(status="applied", applied={"at": "2026-09-01 10:00", "sha": "abc", "file": "SOUL.md"}) + raw = _raw_finding(evidence=[{"session": "ws", "when": "2026-09-02 13:53"}]) + merged = reflect_auto.merge_findings([applied], _parsed(_answer(raw)), TODAY) + + assert merged[0].regression_of == "f0001" + assert merged[0].status == reflect_auto.STATUS_OPEN + + def test_rejected_pattern_never_opens_again(self): + """Rejection is a decision, not a deferral — a repeat must not start nagging again.""" + raw = _raw_finding(occurrences=9, sessions_affected=5) + merged = reflect_auto.merge_findings([_filed(status="rejected")], _parsed(_answer(raw)), TODAY) + assert merged[0].status == reflect_auto.STATUS_WATCH + + def test_rejection_survives_the_run_after_next(self): + """The watch record filed after a rejection is newer — it must not supersede the decision.""" + store = [_filed(status="rejected")] + raw = _raw_finding(occurrences=9, sessions_affected=5) + + first = reflect_auto.merge_findings(store, _parsed(_answer(raw)), "2026-09-02") + second = reflect_auto.merge_findings(reflect_auto.supersede(store, first), _parsed(_answer(raw)), "2026-09-03") + assert second[0].status == reflect_auto.STATUS_WATCH + + def test_rejected_pattern_does_not_inherit_counts(self): + merged = reflect_auto.merge_findings([_filed(status="rejected")], _parsed(_answer(_raw_finding())), TODAY) + assert merged[0].occurrences == 1 + + def test_regression_flag_survives_a_second_sighting(self): + """A pattern that came back after a fix stays flagged until it is dealt with.""" + applied = [_filed(status="applied")] + first = reflect_auto.merge_findings(applied, _parsed(_answer(_raw_finding())), "2026-09-02") + second = reflect_auto.merge_findings( + reflect_auto.supersede(applied, first), _parsed(_answer(_raw_finding())), "2026-09-03" + ) + assert second[0].regression_of == "f0001" + + def test_ids_are_unique_against_existing_records(self): + merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding(pattern="other-thing"))), TODAY) + assert merged[0].id != "f0001" + + +class TestLastSeen: + """`created` is when a record was filed; only the evidence says when the pattern last occurred.""" + + def _merge(self, *evidence: dict) -> reflect_auto.Finding: + raw = _raw_finding(evidence=list(evidence)) + return reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)[0] + + def test_the_newest_evidence_date_wins(self): + finding = self._merge( + {"session": "ws_a", "when": "2026-08-20"}, + {"session": "ws_b", "when": "2026-08-31 13:53"}, + {"session": "ws_c", "when": "2026-08-25"}, + ) + assert finding.last_seen == "2026-08-31", "mixed shapes, only the leading date counts" + + def test_undated_evidence_falls_back_to_created(self): + finding = self._merge({"session": "ws_a", "excerpt": "no when at all"}) + assert finding.last_seen == TODAY + + def test_non_date_when_is_ignored(self): + finding = self._merge({"session": "ws_a", "when": "yesterday"}, {"session": "ws_b", "when": "turn 4"}) + assert finding.last_seen == TODAY + + def test_folded_evidence_from_an_earlier_run_counts_too(self): + existing = [_filed(evidence=[{"session": "ws_old", "when": "2026-09-05"}])] + raw = _raw_finding(evidence=[{"session": "ws_new", "when": "2026-08-01"}]) + merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY) + assert merged[0].last_seen == "2026-09-05" + + def test_last_seen_is_stored_for_the_review_to_sort_by(self): + assert self._merge({"session": "ws_a", "when": "2026-08-20"}).to_json()["last_seen"] == "2026-08-20" + + +class TestSupersede: + def test_folded_record_is_replaced_not_duplicated(self): + existing = [_filed()] + merged = reflect_auto.merge_findings(existing, _parsed(_answer(_raw_finding())), TODAY) + result = reflect_auto.supersede(existing, merged) + assert len(result) == 1 + assert result[0]["status"] == reflect_auto.STATUS_OPEN + + def test_unrelated_records_survive(self): + existing = [_filed(id="f0001", pattern="other-thing", status="applied")] + merged = reflect_auto.merge_findings(existing, _parsed(_answer(_raw_finding())), TODAY) + result = reflect_auto.supersede(existing, merged) + assert {record["pattern"] for record in result} == {"other-thing", "retry-without-diagnosis"} + + def test_applied_history_is_kept_for_regression_tracking(self): + existing = [_filed(status="applied")] + merged = reflect_auto.merge_findings(existing, _parsed(_answer(_raw_finding())), TODAY) + result = reflect_auto.supersede(existing, merged) + assert len(result) == 2 + + +class TestRendering: + def test_known_patterns_feed_the_id_vocabulary_back(self): + text = reflect_auto._known_patterns([_filed()]) + assert "`retry-without-diagnosis`" in text + assert "[watch]" in text + + def test_known_patterns_handles_an_empty_store(self): + assert "none yet" in reflect_auto._known_patterns([]) + + def test_report_marks_regressions(self): + merged = reflect_auto.merge_findings([_filed(status="applied")], _parsed(_answer(_raw_finding())), TODAY) + report = reflect_auto.render_report(merged, _stats(), None, WHEN) + assert "REGRESSION" in report + + def test_report_renders_a_patch_as_a_diff(self): + raw = _raw_finding(patch={"file": "a.md", "old_text": "try again", "new_text": "STOP"}) + merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY) + report = reflect_auto.render_report(merged, _stats(), None, WHEN) + assert "```diff" in report + assert "- try again" in report + assert "+ STOP" in report + + def test_report_shows_how_old_a_finding_is(self): + """`occurrences` is cumulative, so the count alone cannot say whether this is still live.""" + merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding())), TODAY) + report = reflect_auto.render_report(merged, _stats(), None, WHEN) + assert "first seen 2026-09-01, last seen 2026-07-11" in report, "the evidence date, not the refile date" + + def test_a_brand_new_finding_reports_one_date(self): + raw = _raw_finding(evidence=[{"session": "websocket_abc", "excerpt": "no date here"}]) + merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY) + report = reflect_auto.render_report(merged, _stats(), None, WHEN) + assert f"first seen {TODAY}, last seen {TODAY}" in report + + def test_report_marks_a_finding_the_window_no_longer_reaches(self): + """Nothing in the window backs it any more, so no future run will refresh it either.""" + raw = _raw_finding(occurrences=7, sessions_affected=4) + merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY) + report = reflect_auto.render_report(merged, _stats(window_from="2026-08-18T00:00:00"), None, WHEN) + assert "STALE" in report + + def test_a_finding_inside_the_window_is_not_stale(self): + raw = _raw_finding(evidence=[{"session": "websocket_abc", "when": "2026-09-01", "excerpt": "x"}]) + merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY) + report = reflect_auto.render_report(merged, _stats(window_from="2026-08-18T00:00:00"), None, WHEN) + assert "STALE" not in report + + def test_empty_report_says_so(self): + report = reflect_auto.render_report([], _stats(), None, WHEN) + assert "Nothing to report" in report + + def test_report_shows_the_known_pattern_rate(self): + """The one number proving the skill pays off — without it results/ is dead weight.""" + report = reflect_auto.render_report([], _stats(repeat_per_100=3.2), 5.1, WHEN) + assert "Known patterns: 3.2 occurrences / 100 sessions (previous run 5.1)." in report + + def test_first_run_has_no_trend_to_show(self): + assert "(minule" not in reflect_auto.render_report([], _stats(), None, WHEN) + + +class TestPromptContract: + def test_prompt_forbids_writing(self): + """The read-only instruction is one of the two guards on the analysis run.""" + prompt = reflect_auto.build_prompt("digest", "none", 3) + assert "Write nothing" in prompt + + def test_prompt_carries_digest_and_known_patterns(self): + prompt = reflect_auto.build_prompt("SESSION-DIGEST-HERE", "PATTERN-LIST-HERE", 3) + assert "SESSION-DIGEST-HERE" in prompt + assert "PATTERN-LIST-HERE" in prompt + + def test_prompt_caps_file_reads(self): + """Every extra tool iteration re-prefills the whole digest — the cap is what keeps a batch cheap.""" + assert "Read at most 2 files" in reflect_auto.build_prompt("digest", "none", 3) + + +class FakeBot: + """Stands in for Nanobot: hands out canned replies and records what it was asked.""" + + def __init__(self, replies: list[Any]): + self._replies = list(replies) + self.calls: list[tuple[str, str]] = [] + + async def run(self, message: str, *, session_key: str): + self.calls.append((message, session_key)) + reply = self._replies.pop(0) if len(self._replies) > 1 else self._replies[0] + if isinstance(reply, Exception): + raise reply + return reply + + +def _reply(content: str = "", stop_reason: str | None = None, error: str | None = None) -> SimpleNamespace: + return SimpleNamespace(content=content, stop_reason=stop_reason, error=error) + + +def _llm_failure() -> SimpleNamespace: + """What nanobot hands back when the provider gives up: the error text as the reply.""" + return _reply("Error calling LLM: timed out after 300s", stop_reason="error", error="timed out after 300s") + + +def _session_records(started: str, turns: int = 3) -> list[dict]: + """A conversation long enough to clear reflect_distill.MIN_MESSAGES.""" + records: list[dict] = [{"_type": "metadata", "key": "websocket:x", "created_at": started}] + for i in range(turns): + records.append({"role": "user", "content": f"dotaz {i}"}) + records.append({"role": "assistant", "content": f"answer {i}"}) + return records + + +EARLIER = "2026-07-11T14:02:03" +LATER = "2026-08-20T09:15:00" + + +@pytest.fixture +def workspace(tmp_path): + """A workspace that is a real git repo with two analysable sessions, like the server's.""" + sessions = tmp_path / "sessions" + sessions.mkdir() + for name, started in (("websocket_first", EARLIER), ("websocket_second", LATER)): + records = _session_records(started) + (sessions / f"{name}.jsonl").write_text( + "\n".join(json.dumps(r, ensure_ascii=False) for r in records), encoding="utf-8" + ) + (tmp_path / "reflect").mkdir() + + for args in ( + ["init", "-q"], + ["config", "user.email", "test@example.com"], + ["config", "user.name", "test"], + ["add", "-A"], + ["commit", "-q", "-m", "init"], + ): + subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True) + return tmp_path + + +def _args(**overrides): + """One session per batch (budget_chars=1), so the two fixture sessions make two batches.""" + args = { + "dry_run": False, + "all": False, + "budget_chars": 1, + "max_batches": 0, + "deadline_minutes": 20, + "window_days": 0, + } + args.update(overrides) + return SimpleNamespace(**args) + + +def _state(workspace: Path) -> dict: + return json.loads((workspace / "reflect" / "state.json").read_text(encoding="utf-8")) + + +def _findings(workspace: Path) -> list[dict]: + path = workspace / "reflect" / "findings.jsonl" + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def _seed_findings(workspace: Path, *records: dict) -> None: + path = workspace / "reflect" / "findings.jsonl" + path.write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in records) + "\n", encoding="utf-8") + + +def _run(workspace: Path, bot, monkeypatch, **arg_overrides) -> tuple[str, int]: + monkeypatch.setattr(reflect_auto, "_open_bot", lambda preset: bot) + return asyncio.run(reflect_auto._run(workspace, _args(**arg_overrides), WHEN)) + + +class TestRunPersistence: + """The 2026-09-02 failure: a 30min timeout threw away two finished batches and kept the cursor.""" + + def test_every_batch_is_filed_as_it_finishes(self, workspace, monkeypatch): + bot = FakeBot( + [ + _reply(_answer(_raw_finding())), + _reply(_answer(_raw_finding(pattern="tool-call-leaked-as-text"))), + ] + ) + _run(workspace, bot, monkeypatch) + + assert {record["pattern"] for record in _findings(workspace)} == { + "retry-without-diagnosis", + "tool-call-leaked-as-text", + } + assert _state(workspace)["cursor"] == LATER + assert [run["batches"] for run in _state(workspace)["runs"]] == [2], "one run entry, updated in place" + + def test_the_same_pattern_in_two_batches_folds_into_one_record(self, workspace, monkeypatch): + """Per-batch filing must not turn cross-batch dedup into duplicate records.""" + bot = FakeBot([_reply(_answer(_raw_finding()))]) + _run(workspace, bot, monkeypatch) + + records = _findings(workspace) + assert len(records) == 1 + assert records[0]["occurrences"] == 2 + assert records[0]["status"] == "open", "the second sighting is what promotes a watched pattern" + + def test_the_report_counts_a_repeated_pattern_once(self, workspace, monkeypatch): + """The report and the Telegram line have to say what the store holds, not how often it was refiled.""" + bot = FakeBot([_reply(_answer(_raw_finding()))]) + message, _ = _run(workspace, bot, monkeypatch) + + report = (workspace / "results" / f"{WHEN:%Y-%m-%d}_reflect.md").read_text(encoding="utf-8") + assert len([line for line in report.splitlines() if line.startswith("## ")]) == 1 + assert "Findings: 1 (1 to review, 0 watched)" in report + assert _state(workspace)["runs"][-1]["open"] == 1 + assert "1 findings to review" in message + + def test_a_batch_that_dies_does_not_take_the_finished_one_with_it(self, workspace, monkeypatch): + bot = FakeBot([_reply(_answer(_raw_finding())), RuntimeError("provider down")]) + with pytest.raises(RuntimeError): + _run(workspace, bot, monkeypatch) + + assert len(_findings(workspace)) == 1 + assert _state(workspace)["cursor"] == EARLIER + assert _state(workspace)["runs"][-1]["batches"] == 1 + + def test_the_cursor_never_moves_backwards(self, workspace, monkeypatch): + """An `--all` walk starts at the oldest session and must not undo the nightly progress.""" + reflect_auto._save_state(workspace, {"cursor": "2026-12-31T00:00:00", "runs": []}) + bot = FakeBot([_reply(_answer(_raw_finding()))]) + _run(workspace, bot, monkeypatch, all=True) + + assert _state(workspace)["cursor"] == "2026-12-31T00:00:00" + + def test_the_deadline_stops_the_run_and_says_so(self, workspace, monkeypatch): + _seed_findings(workspace, _filed()) + bot = FakeBot([_reply(_answer(_raw_finding()))]) + message, _ = _run(workspace, bot, monkeypatch, deadline_minutes=0) + + assert len(bot.calls) == 1 + assert _state(workspace)["cursor"] == EARLIER + assert "Analysed 1/2 batches." in message + + def test_a_finished_run_reports_no_partial_progress(self, workspace, monkeypatch): + _seed_findings(workspace, _filed()) + bot = FakeBot([_reply(_answer(_raw_finding()))]) + message, _ = _run(workspace, bot, monkeypatch) + + assert "batches" not in message + + def test_an_unfinished_run_speaks_up_even_with_no_findings(self, workspace, monkeypatch): + """Silence here is how a starved cursor went unnoticed for three nights.""" + bot = FakeBot([_reply(_answer(_raw_finding()))]) + message, _ = _run(workspace, bot, monkeypatch, deadline_minutes=0) + + assert "1 batches left" in message + + def test_a_finished_run_with_no_findings_stays_quiet(self, workspace, monkeypatch): + bot = FakeBot([_reply(_answer())]) + message, _ = _run(workspace, bot, monkeypatch) + + assert message == "" + + +class TestWindow: + """Findings must describe recent behaviour; a months-old backlog must not starve the run.""" + + def test_the_window_skips_everything_older(self, workspace, monkeypatch): + bot = FakeBot([_reply(_answer(_raw_finding()))]) + _run(workspace, bot, monkeypatch, window_days=21) + + assert len(bot.calls) == 1, "only the session inside the window is analysable" + assert "websocket_second" in bot.calls[0][0] + assert "websocket_first" not in bot.calls[0][0] + assert _state(workspace)["cursor"] == LATER, "the skipped backlog is skipped for good" + + def test_the_cursor_still_wins_over_the_window(self, workspace, monkeypatch): + """The window is a ceiling, not a rewind — nothing already counted may be re-read.""" + reflect_auto._save_state(workspace, {"cursor": LATER, "runs": []}) + bot = FakeBot([_reply(_answer(_raw_finding()))]) + message, sessions = _run(workspace, bot, monkeypatch, window_days=365) + + assert bot.calls == [] + assert (message, sessions) == ("", 0) + + def test_the_report_says_what_the_window_covered(self, workspace, monkeypatch): + bot = FakeBot([_reply(_answer(_raw_finding()))]) + _run(workspace, bot, monkeypatch, window_days=21) + + report = (workspace / "results" / f"{WHEN:%Y-%m-%d}_reflect.md").read_text(encoding="utf-8") + assert "Window: from 2026-08-18, batches 1/1." in report + + def test_a_live_all_run_is_refused(self, workspace): + """`--all` re-reads counted sessions and merge_findings would sum their occurrences in.""" + with pytest.raises(SystemExit): + reflect_auto.main(["--all", "--workspace", str(workspace)]) + + def test_a_dry_all_run_is_allowed(self, workspace): + assert reflect_auto.main(["--all", "--dry-run", "--workspace", str(workspace)]) == 0 + assert list((workspace / "tmp").glob("reflect-batch.*.md")) + + +class TestLlmErrorHandling: + """A dead provider is not a malformed answer; conflating the two burned a validator attempt.""" + + def test_a_provider_failure_re_asks_the_original_question(self, workspace, monkeypatch): + bot = FakeBot([_llm_failure(), _reply(_answer(_raw_finding()))]) + parsed = asyncio.run(reflect_auto._resolve_findings(bot, "reflect:test-0", "PROMPT", workspace, 4)) + + assert len(parsed) == 1 + first, second = bot.calls + assert second[0] == "PROMPT", "the retry must re-ask, not blame the model for the error text" + assert second[1] != first[1], "the failed turn poisoned that session; retry needs a fresh key" + + def test_a_dead_provider_is_reported_as_such(self, workspace, monkeypatch): + bot = FakeBot([_llm_failure()]) + with pytest.raises(reflect_auto.ReflectError, match="model unavailable"): + asyncio.run(reflect_auto._resolve_findings(bot, "reflect:test-0", "PROMPT", workspace, 4)) + + assert len(bot.calls) == reflect_auto.MAX_LLM_ERROR_RETRIES + 1 + + def test_a_malformed_answer_still_gets_validator_feedback(self, workspace, monkeypatch): + bot = FakeBot([_reply("no json here"), _reply(_answer(_raw_finding()))]) + parsed = asyncio.run(reflect_auto._resolve_findings(bot, "reflect:test-0", "PROMPT", workspace, 4)) + + assert len(parsed) == 1 + assert "rejected by the validator" in bot.calls[1][0] diff --git a/skills/reflect/tests/test_reflect_distill.py b/skills/reflect/tests/test_reflect_distill.py new file mode 100644 index 0000000..37dc9b6 --- /dev/null +++ b/skills/reflect/tests/test_reflect_distill.py @@ -0,0 +1,241 @@ +"""Tests for reflect_distill.py — mechanical distillation of session logs.""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import reflect_distill + + +def _write_session(directory: Path, name: str, records: list[dict]) -> Path: + path = directory / f"{name}.jsonl" + path.write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in records), encoding="utf-8") + return path + + +def _conversation(turns: int = 3) -> list[dict]: + """A session long enough to clear MIN_MESSAGES.""" + records: list[dict] = [{"_type": "metadata", "key": "websocket:x", "created_at": "2026-07-11T14:02:03"}] + for i in range(turns): + records.append({"role": "user", "content": f"dotaz {i}"}) + records.append({"role": "assistant", "content": f"answer {i}"}) + return records + + +class TestNoiseFilter: + @pytest.mark.parametrize( + "name", + [ + "reflect_20260908-033000", + "compact-memory-auto_20260715-054147", + "detach_abc", + "dream_xyz", + "cron_e81cda77", + "cli_kimi-ollama-test", + "wiki-compile", + "note-compile", + ], + ) + def test_machinery_prefixes_are_noise(self, name): + assert reflect_distill.is_noise(name) + + @pytest.mark.parametrize("name", ["wiki-capture-test2", "test-note-search-zzz"]) + def test_prefix_variants_are_noise(self, name): + """Exact matching used to let `…test2` through; the filter matches prefixes.""" + assert reflect_distill.is_noise(name) + + @pytest.mark.parametrize("name", ["websocket_e5a6aac3-2c0a", "telegram_8826147089"]) + def test_real_conversations_are_kept(self, name): + assert not reflect_distill.is_noise(name) + + def test_base64_session_names_are_decoded(self): + """Older sessions are stored base64-encoded; `ZHJlYW06…` is `dream:…`.""" + assert reflect_distill.is_noise("ZHJlYW06MjAyNjA4MzEtMTA0NjM2") + assert not reflect_distill.is_noise("d2Vic29ja2V0OjUyZDBmMzM4") + + def test_reflect_excludes_its_own_sessions(self): + """Without this the skill would analyse its own runs on the next pass.""" + assert reflect_distill.is_noise("reflect_20260908-033000") + + +class TestDistillSession: + def test_short_sessions_are_skipped(self, tmp_path): + path = _write_session(tmp_path, "websocket_short", [{"role": "user", "content": "ahoj"}]) + assert reflect_distill.distill_session(path) is None + + def test_noise_sessions_are_skipped(self, tmp_path): + path = _write_session(tmp_path, "dream_nightly", _conversation()) + assert reflect_distill.distill_session(path) is None + + def test_header_carries_key_time_and_count(self, tmp_path): + path = _write_session(tmp_path, "websocket_abc", _conversation()) + digest = reflect_distill.distill_session(path) + assert digest.text.startswith(reflect_distill.SESSION_MARKER) + assert "websocket_abc" in digest.text + assert "2026-07-11 14:02" in digest.text + assert digest.message_count == 6 + + def test_header_is_not_a_markdown_heading(self, tmp_path): + """Assistant prose is full of `###`, so a heading would not read as a boundary.""" + records = _conversation() + records.append({"role": "assistant", "content": "### Summary\ndone"}) + path = _write_session(tmp_path, "websocket_abc", records) + digest = reflect_distill.distill_session(path) + assert not digest.text.startswith("#") + assert digest.text.count(reflect_distill.SESSION_MARKER) == 1 + + def test_user_and_assistant_prose_is_kept_whole(self, tmp_path): + prose = "this is a long answer " * 40 + records = _conversation() + records.append({"role": "assistant", "content": prose}) + path = _write_session(tmp_path, "websocket_abc", records) + assert prose.strip() in reflect_distill.distill_session(path).text + + def test_slash_commands_are_dropped(self, tmp_path): + records = _conversation() + records.append({"role": "user", "content": "/model", "_command": True}) + path = _write_session(tmp_path, "websocket_abc", records) + assert "/model" not in reflect_distill.distill_session(path).text + + def test_decoded_key_is_used_as_name(self, tmp_path): + path = _write_session(tmp_path, "d2Vic29ja2V0OjUyZDBmMzM4", _conversation()) + assert reflect_distill.distill_session(path).name == "websocket:52d0f338" + + +class TestToolRendering: + def _digest_with_tool(self, tmp_path, call, result): + records = _conversation() + records.append({"role": "assistant", "content": "", "tool_calls": [call]}) + records.append(result) + path = _write_session(tmp_path, "websocket_abc", records) + return reflect_distill.distill_session(path).text + + def test_tool_result_body_is_replaced_by_metadata(self, tmp_path): + """Tool results are 89% of the corpus and carry almost no diagnostic value.""" + body = "x" * 5000 + text = self._digest_with_tool( + tmp_path, + {"id": "c1", "function": {"name": "read_file", "arguments": '{"path": "/tmp/a.md"}'}}, + {"role": "tool", "tool_call_id": "c1", "name": "read_file", "content": body}, + ) + assert body not in text + assert "read_file: ok, 5.0 kB" in text + + def test_errors_are_flagged_with_their_message(self, tmp_path): + text = self._digest_with_tool( + tmp_path, + {"id": "c1", "function": {"name": "web_fetch", "arguments": '{"url": "https://x.dev"}'}}, + {"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "Error: 403 Forbidden"}, + ) + assert "ERROR Error: 403 Forbidden" in text + + def test_workspace_path_prefix_is_stripped(self, tmp_path): + text = self._digest_with_tool( + tmp_path, + { + "id": "c1", + "function": { + "name": "read_file", + "arguments": '{"path": "/home/nanobot/.nanobot/workspace/skills/note/SKILL.md"}', + }, + }, + {"role": "tool", "tool_call_id": "c1", "name": "read_file", "content": "ok"}, + ) + assert "path=skills/note/SKILL.md" in text + + def test_malformed_arguments_still_render(self, tmp_path): + """A truncated argument blob is itself a signal worth seeing, not a crash.""" + text = self._digest_with_tool( + tmp_path, + {"id": "c1", "function": {"name": "exec", "arguments": '{"command": "ls'}}, + {"role": "tool", "tool_call_id": "c1", "name": "exec", "content": "ok"}, + ) + assert "exec(" in text + + def test_long_urls_keep_their_distinguishing_tail(self, tmp_path): + """Front-truncation made different fetches look identical and hid real retries.""" + base = "https://raw.githubusercontent.com/some-owner/some-repo/main/packages/core/src/" + first = self._digest_with_tool( + tmp_path, + {"id": "c1", "function": {"name": "web_fetch", "arguments": json.dumps({"url": base + "alpha.ts"})}}, + {"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "ok"}, + ) + second = self._digest_with_tool( + tmp_path, + {"id": "c1", "function": {"name": "web_fetch", "arguments": json.dumps({"url": base + "omega.ts"})}}, + {"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "ok"}, + ) + assert "alpha.ts" in first + assert "omega.ts" in second + assert first != second + + def test_repeated_identical_calls_render_identically(self, tmp_path): + """The LLM detects retry loops by seeing the same line twice — so it must match.""" + call = {"id": "c1", "function": {"name": "web_fetch", "arguments": '{"url": "https://x.dev/a"}'}} + records = _conversation() + for _ in range(2): + records.append({"role": "assistant", "content": "", "tool_calls": [call]}) + records.append({"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "Error: 403"}) + path = _write_session(tmp_path, "websocket_abc", records) + lines = [line for line in reflect_distill.distill_session(path).text.splitlines() if "→ web_fetch" in line] + assert len(lines) == 2 + assert lines[0] == lines[1] + + def test_reasoning_is_shortened_not_dropped(self, tmp_path): + records = _conversation() + records.append({"role": "assistant", "content": "ok", "reasoning_content": "reasoning " * 200}) + path = _write_session(tmp_path, "websocket_abc", records) + text = reflect_distill.distill_session(path).text + assert "~ reasoning" in text + assert len(text) < 2000 + + +class TestBatching: + def test_batches_respect_the_character_budget(self, tmp_path): + for i in range(6): + _write_session(tmp_path, f"websocket_{i}", _conversation(turns=8)) + paths = reflect_distill.collect_sessions(tmp_path) + batches = list(reflect_distill.iter_batches(paths, budget_chars=400)) + assert len(batches) > 1 + assert all(batch for batch in batches) + + def test_oversized_session_is_not_split_or_dropped(self, tmp_path): + """Truncating a huge session would hide exactly the runaway loops worth finding.""" + records = _conversation() + records.append({"role": "assistant", "content": "y" * 5000}) + _write_session(tmp_path, "websocket_big", records) + batches = list(reflect_distill.iter_batches(reflect_distill.collect_sessions(tmp_path), budget_chars=100)) + assert len(batches) == 1 + assert len(batches[0][0]) > 5000 + + def test_collect_sessions_skips_noise(self, tmp_path): + _write_session(tmp_path, "websocket_keep", _conversation()) + _write_session(tmp_path, "dream_drop", _conversation()) + names = [p.stem for p in reflect_distill.collect_sessions(tmp_path)] + assert names == ["websocket_keep"] + + def test_cursor_written_by_a_run_excludes_those_sessions_next_time(self, tmp_path): + """The round trip reflect_auto actually makes: digest.started becomes the next `since`. + + A display-formatted cursor (`2026-07-11 14:02`) compares wrong against the raw ISO in + the log, because `T` sorts above a space — every session of that day came back. + """ + _write_session(tmp_path, "websocket_done", _conversation()) + digests = [ + d + for batch in reflect_distill.iter_batches(reflect_distill.collect_sessions(tmp_path), 10**6) + for d in batch + ] + cursor = max(digest.started for digest in digests) + assert reflect_distill.collect_sessions(tmp_path, since=cursor) == [] + + def test_since_excludes_already_processed_sessions(self, tmp_path): + _write_session(tmp_path, "websocket_old", _conversation()) + new = _conversation() + new[0]["created_at"] = "2026-08-20T09:00:00" + _write_session(tmp_path, "websocket_new", new) + names = [p.stem for p in reflect_distill.collect_sessions(tmp_path, since="2026-08-01T00:00:00")] + assert names == ["websocket_new"]