upravy projektu a skillu
This commit is contained in:
lachtan
381
skills/reflect/tests/test_reflect_apply.py
Normal file
381
skills/reflect/tests/test_reflect_apply.py
Normal file
@@ -0,0 +1,381 @@
|
||||
"""Tests for reflect_apply.py — the gate that stands between a finding and a real edit.
|
||||
|
||||
These are the guarantees the user asked for: nothing is applied without an explicit,
|
||||
per-finding approval, an ambiguous patch is refused rather than guessed at, and every
|
||||
applied change is one revertable commit touching one file.
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
import reflect_apply
|
||||
|
||||
NOW = datetime(2026, 9, 8, 10, 30)
|
||||
TARGET_REL = "skills/demo/SKILL.md"
|
||||
ORIGINAL = "# Demo\n\nIf the fetch fails, try again.\n\nDone.\n"
|
||||
|
||||
|
||||
def _record(**overrides):
|
||||
record = {
|
||||
"id": "f7a2",
|
||||
"status": "open",
|
||||
"created": "2026-09-08",
|
||||
"pattern": "retry-without-diagnosis",
|
||||
"severity": "medium",
|
||||
"diagnosis": "Repeats a call without diagnosing it.",
|
||||
"evidence": [{"session": "websocket_abc"}],
|
||||
"occurrences": 7,
|
||||
"sessions_affected": 4,
|
||||
"proposal": "Add a hard STOP gate.",
|
||||
"patch": {
|
||||
"file": TARGET_REL,
|
||||
"old_text": "If the fetch fails, try again.",
|
||||
"new_text": "If the fetch fails, STOP and diagnose.",
|
||||
},
|
||||
}
|
||||
record.update(overrides)
|
||||
return record
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path):
|
||||
"""A miniature workspace that is a real git repo, like the server's."""
|
||||
target = tmp_path / TARGET_REL
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(ORIGINAL, encoding="utf-8")
|
||||
(tmp_path / "reflect").mkdir()
|
||||
(tmp_path / "reflect" / "findings.jsonl").write_text(json.dumps(_record()) + "\n", encoding="utf-8")
|
||||
|
||||
for args in (
|
||||
["init", "-q"],
|
||||
["config", "user.email", "test@example.com"],
|
||||
["config", "user.name", "test"],
|
||||
["add", "-A"],
|
||||
["commit", "-q", "-m", "init"],
|
||||
):
|
||||
subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _write_findings(workspace, *records):
|
||||
path = workspace / "reflect" / "findings.jsonl"
|
||||
path.write_text("\n".join(json.dumps(r) for r in records) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _findings(workspace):
|
||||
lines = (workspace / "reflect" / "findings.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
return [json.loads(line) for line in lines if line.strip()]
|
||||
|
||||
|
||||
def _git(workspace, *args):
|
||||
return subprocess.run(["git", *args], cwd=workspace, capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
|
||||
class TestRefusals:
|
||||
def test_missing_original_text_is_refused(self, workspace):
|
||||
_write_findings(workspace, _record(patch={**_record()["patch"], "old_text": "text that is not there"}))
|
||||
with pytest.raises(reflect_apply.ApplyError, match="no longer in"):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL
|
||||
|
||||
def test_ambiguous_original_text_is_refused(self, workspace):
|
||||
"""Two matches means the intended location is a guess — refuse, never guess."""
|
||||
(workspace / TARGET_REL).write_text(ORIGINAL + "If the fetch fails, try again.\n", encoding="utf-8")
|
||||
before = (workspace / TARGET_REL).read_text(encoding="utf-8")
|
||||
with pytest.raises(reflect_apply.ApplyError, match="occurs 2"):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == before
|
||||
|
||||
def test_a_finding_without_a_patch_is_refused(self, workspace):
|
||||
record = _record()
|
||||
del record["patch"]
|
||||
_write_findings(workspace, record)
|
||||
with pytest.raises(reflect_apply.ApplyError, match="no patch"):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
|
||||
@pytest.mark.parametrize("status", ["watch", "applied", "rejected"])
|
||||
def test_only_open_findings_can_be_applied(self, workspace, status):
|
||||
"""`watch` findings were never shown to the user, so they were never approved."""
|
||||
_write_findings(workspace, _record(status=status))
|
||||
with pytest.raises(reflect_apply.ApplyError):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
|
||||
def test_unknown_id_is_refused(self, workspace):
|
||||
with pytest.raises(reflect_apply.ApplyError, match="no finding"):
|
||||
reflect_apply.apply_finding(workspace, "nope", None, NOW)
|
||||
|
||||
def test_path_outside_the_workspace_is_refused(self, workspace):
|
||||
_write_findings(workspace, _record(patch={**_record()["patch"], "file": "../../../etc/passwd"}))
|
||||
with pytest.raises(reflect_apply.ApplyError):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
|
||||
def test_refusal_leaves_the_store_untouched(self, workspace):
|
||||
_write_findings(workspace, _record(patch={**_record()["patch"], "old_text": "absent"}))
|
||||
with pytest.raises(reflect_apply.ApplyError):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert _findings(workspace)[0]["status"] == "open"
|
||||
|
||||
|
||||
class TestApply:
|
||||
def test_patch_is_applied_and_committed(self, workspace):
|
||||
message = reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert "STOP and diagnose" in (workspace / TARGET_REL).read_text(encoding="utf-8")
|
||||
assert "commit" in message
|
||||
assert "retry-without-diagnosis (f7a2)" in _git(workspace, "log", "-1", "--pretty=%s")
|
||||
|
||||
def test_commit_contains_only_the_patched_file(self, workspace):
|
||||
"""Dream and other skills leave unrelated work in progress — it must not be swept in."""
|
||||
(workspace / "unrelated.md").write_text("someone else's work\n", encoding="utf-8")
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert _git(workspace, "show", "--name-only", "--pretty=", "HEAD") == TARGET_REL
|
||||
assert "unrelated.md" in _git(workspace, "status", "--porcelain")
|
||||
|
||||
def test_prior_edits_to_the_same_file_are_checkpointed_first(self, workspace):
|
||||
"""The patch commit must be the patch alone, so the revert is exact."""
|
||||
target = workspace / TARGET_REL
|
||||
target.write_text(ORIGINAL + "\nhand-written note\n", encoding="utf-8")
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
subjects = _git(workspace, "log", "-2", "--pretty=%s").splitlines()
|
||||
assert subjects[1] == "reflect: checkpoint before f7a2"
|
||||
assert "hand-written note" in target.read_text(encoding="utf-8")
|
||||
|
||||
def test_revert_restores_the_original(self, workspace):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
sha = _findings(workspace)[0]["applied"]["sha"]
|
||||
subprocess.run(["git", "revert", "--no-edit", sha], cwd=workspace, check=True, capture_output=True)
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL
|
||||
|
||||
def test_store_records_status_sha_and_file(self, workspace):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
record = _findings(workspace)[0]
|
||||
assert record["status"] == "applied"
|
||||
assert record["applied"]["file"] == TARGET_REL
|
||||
assert record["applied"]["sha"]
|
||||
|
||||
def test_other_records_survive_byte_identical(self, workspace):
|
||||
other = _record(id="f0002", pattern="other-thing", status="watch")
|
||||
_write_findings(workspace, _record(), other)
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert _findings(workspace)[1] == other
|
||||
|
||||
def test_audit_line_is_appended(self, workspace):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
log = (workspace / "log" / "reflect.log").read_text(encoding="utf-8")
|
||||
assert "APPLIED f7a2 [retry-without-diagnosis]" in log
|
||||
|
||||
def test_user_edited_replacement_is_used_and_recorded(self, workspace):
|
||||
reflect_apply.apply_finding(workspace, "f7a2", "STOP. Read the status code first.", NOW)
|
||||
assert "Read the status code first." in (workspace / TARGET_REL).read_text(encoding="utf-8")
|
||||
assert _findings(workspace)[0]["applied"]["edited_by_user"] is True
|
||||
|
||||
|
||||
class TestCheckAndReject:
|
||||
def test_check_reports_success_without_touching_anything(self, workspace):
|
||||
exit_code = reflect_apply.main(["--id", "f7a2", "--check", "--workspace", str(workspace)])
|
||||
assert exit_code == 0
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL
|
||||
assert _findings(workspace)[0]["status"] == "open"
|
||||
|
||||
def test_check_prints_the_diff_the_skill_shows(self, workspace, capsys):
|
||||
"""The skill presents whatever this prints, so the diff has to come from here."""
|
||||
reflect_apply.main(["--id", "f7a2", "--check", "--workspace", str(workspace)])
|
||||
printed = capsys.readouterr().out
|
||||
assert "-If the fetch fails, try again." in printed
|
||||
assert "+If the fetch fails, STOP and diagnose." in printed
|
||||
|
||||
def test_check_fails_on_a_stale_patch(self, workspace):
|
||||
_write_findings(workspace, _record(patch={**_record()["patch"], "old_text": "absent"}))
|
||||
assert reflect_apply.main(["--id", "f7a2", "--check", "--workspace", str(workspace)]) == 2
|
||||
|
||||
def test_failed_commit_restores_the_file(self, workspace, monkeypatch, capsys):
|
||||
"""Exit code 2 tells the skill nothing happened — so nothing may be left behind."""
|
||||
real_git = reflect_apply._git
|
||||
|
||||
def failing_git(ws, *args):
|
||||
if args[0] == "commit" and "checkpoint" not in " ".join(args):
|
||||
raise subprocess.CalledProcessError(1, ["git", *args], stderr="empty ident name")
|
||||
return real_git(ws, *args)
|
||||
|
||||
monkeypatch.setattr(reflect_apply, "_git", failing_git)
|
||||
exit_code = reflect_apply.main(["--id", "f7a2", "--workspace", str(workspace)])
|
||||
|
||||
assert exit_code == 2
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL
|
||||
assert _findings(workspace)[0]["status"] == "open"
|
||||
assert _git(workspace, "status", "--porcelain") == ""
|
||||
|
||||
def test_reject_closes_the_finding_without_editing(self, workspace):
|
||||
reflect_apply.reject_finding(workspace, "f7a2", "false positive", NOW)
|
||||
assert _findings(workspace)[0]["status"] == "rejected"
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL
|
||||
assert "REJECTED f7a2" in (workspace / "log" / "reflect.log").read_text(encoding="utf-8")
|
||||
|
||||
def test_cli_refusal_exits_nonzero(self, workspace):
|
||||
assert reflect_apply.main(["--id", "unknown", "--workspace", str(workspace)]) == 2
|
||||
|
||||
|
||||
class TestSetPatch:
|
||||
"""A patch drafted during the review is filed by the script — the agent never touches the store.
|
||||
|
||||
Before this existed the agent had to edit findings.jsonl with an ad-hoc script to get a patch
|
||||
in, which put an LLM inside the audit trail and left unapplicable patches behind on a miss.
|
||||
"""
|
||||
|
||||
def _patch_file(self, tmp_path, **overrides):
|
||||
patch = {"file": TARGET_REL, "old_text": "Done.", "new_text": "Done, and diagnosed."}
|
||||
patch.update(overrides)
|
||||
path = tmp_path / "patch.json"
|
||||
path.write_text(json.dumps(patch), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def _set(self, workspace, path, finding_id="f7a2"):
|
||||
return reflect_apply.main(["--id", finding_id, "--set-patch", str(path), "--workspace", str(workspace)])
|
||||
|
||||
def _without_patch(self, workspace):
|
||||
record = _record()
|
||||
del record["patch"]
|
||||
_write_findings(workspace, record)
|
||||
return record
|
||||
|
||||
def test_a_patch_that_does_not_apply_never_reaches_the_store(self, workspace, tmp_path):
|
||||
"""Verification runs before the write, so a failed attempt leaves nothing behind."""
|
||||
self._without_patch(workspace)
|
||||
assert self._set(workspace, self._patch_file(tmp_path, old_text="TEXT THAT IS NOT THERE")) == 2
|
||||
assert "patch" not in _findings(workspace)[0]
|
||||
|
||||
def test_an_ambiguous_patch_never_reaches_the_store(self, workspace, tmp_path):
|
||||
(workspace / TARGET_REL).write_text(ORIGINAL + "Done.\n", encoding="utf-8")
|
||||
self._without_patch(workspace)
|
||||
assert self._set(workspace, self._patch_file(tmp_path)) == 2
|
||||
assert "patch" not in _findings(workspace)[0]
|
||||
|
||||
def test_a_valid_patch_is_filed_with_its_provenance(self, workspace, tmp_path):
|
||||
self._without_patch(workspace)
|
||||
assert self._set(workspace, self._patch_file(tmp_path)) == 0
|
||||
|
||||
record = _findings(workspace)[0]
|
||||
assert record["patch"]["new_text"] == "Done, and diagnosed."
|
||||
assert record["patch_drafted_at"], "drafted during the review, not proposed by the analysis"
|
||||
assert record["status"] == "open", "drafting a patch decides nothing"
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL, "nothing is applied yet"
|
||||
assert "DRAFTED f7a2 [retry-without-diagnosis]" in (workspace / "log" / "reflect.log").read_text("utf-8")
|
||||
|
||||
def test_the_diff_comes_back_in_the_same_call(self, workspace, tmp_path, capsys):
|
||||
"""Two calls instead of six: draft and show. Asking --check afterwards would be a third."""
|
||||
self._without_patch(workspace)
|
||||
self._set(workspace, self._patch_file(tmp_path))
|
||||
printed = capsys.readouterr().out
|
||||
assert "-Done." in printed
|
||||
assert "+Done, and diagnosed." in printed
|
||||
|
||||
def test_a_wrong_first_draft_can_be_replaced(self, workspace, tmp_path):
|
||||
"""The guard is the `open` status, not the absence of a patch — first drafts get it wrong."""
|
||||
assert self._set(workspace, self._patch_file(tmp_path)) == 0
|
||||
assert _findings(workspace)[0]["patch"]["old_text"] == "Done."
|
||||
|
||||
@pytest.mark.parametrize("status", ["watch", "applied", "rejected"])
|
||||
def test_only_open_findings_can_be_drafted_for(self, workspace, tmp_path, status):
|
||||
_write_findings(workspace, _record(status=status))
|
||||
assert self._set(workspace, self._patch_file(tmp_path)) == 2
|
||||
|
||||
def test_a_patch_outside_the_workspace_is_refused(self, workspace, tmp_path):
|
||||
self._without_patch(workspace)
|
||||
assert self._set(workspace, self._patch_file(tmp_path, file="../../../etc/passwd")) == 2
|
||||
assert "patch" not in _findings(workspace)[0]
|
||||
|
||||
@pytest.mark.parametrize("body", ['{"file": "a.md"}', '{"file": 1, "old_text": "a", "new_text": "b"}', "not json"])
|
||||
def test_a_malformed_patch_file_is_refused(self, workspace, tmp_path, body):
|
||||
self._without_patch(workspace)
|
||||
path = tmp_path / "patch.json"
|
||||
path.write_text(body, encoding="utf-8")
|
||||
assert self._set(workspace, path) == 2
|
||||
assert "patch" not in _findings(workspace)[0]
|
||||
|
||||
@pytest.mark.parametrize("other", [["--check"], ["--skip"], ["--reject", "--reason", "no"]])
|
||||
def test_set_patch_excludes_the_other_decisions(self, workspace, tmp_path, other):
|
||||
path = self._patch_file(tmp_path)
|
||||
with pytest.raises(SystemExit):
|
||||
reflect_apply.main(["--id", "f7a2", "--set-patch", str(path), *other, "--workspace", str(workspace)])
|
||||
|
||||
def test_a_drafted_patch_can_then_be_applied(self, workspace, tmp_path):
|
||||
self._without_patch(workspace)
|
||||
self._set(workspace, self._patch_file(tmp_path))
|
||||
reflect_apply.apply_finding(workspace, "f7a2", None, NOW)
|
||||
assert "Done, and diagnosed." in (workspace / TARGET_REL).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class TestDecisionRecord:
|
||||
"""A decision the audit cannot reconstruct is not recorded — the reason and the skips too."""
|
||||
|
||||
def _log(self, workspace) -> str:
|
||||
return (workspace / "log" / "reflect.log").read_text(encoding="utf-8")
|
||||
|
||||
def test_reject_needs_a_reason(self, workspace):
|
||||
with pytest.raises(SystemExit):
|
||||
reflect_apply.main(["--id", "f7a2", "--reject", "--workspace", str(workspace)])
|
||||
assert _findings(workspace)[0]["status"] == "open"
|
||||
|
||||
def test_a_blank_reason_does_not_count(self, workspace):
|
||||
with pytest.raises(SystemExit):
|
||||
reflect_apply.main(["--id", "f7a2", "--reject", "--reason", " ", "--workspace", str(workspace)])
|
||||
assert _findings(workspace)[0]["status"] == "open"
|
||||
|
||||
def test_a_reason_without_reject_is_refused(self, workspace):
|
||||
with pytest.raises(SystemExit):
|
||||
reflect_apply.main(["--id", "f7a2", "--reason", "because", "--workspace", str(workspace)])
|
||||
|
||||
def test_the_reason_lands_in_the_record_and_the_log(self, workspace):
|
||||
exit_code = reflect_apply.main(
|
||||
["--id", "f7a2", "--reject", "--reason", "false positive", "--workspace", str(workspace)]
|
||||
)
|
||||
assert exit_code == 0
|
||||
rejected = _findings(workspace)[0]["rejected"]
|
||||
assert rejected["reason"] == "false positive"
|
||||
assert rejected["at"]
|
||||
assert "false positive" in self._log(workspace)
|
||||
|
||||
def test_skip_counts_up_and_decides_nothing(self, workspace):
|
||||
for expected in (1, 2, 3):
|
||||
assert reflect_apply.main(["--id", "f7a2", "--skip", "--workspace", str(workspace)]) == 0
|
||||
record = _findings(workspace)[0]
|
||||
assert record["skipped"]["count"] == expected
|
||||
assert record["status"] == "open", "a skip is a deferral, not a decision"
|
||||
|
||||
assert (workspace / TARGET_REL).read_text(encoding="utf-8") == ORIGINAL
|
||||
assert "SKIPPED f7a2 [retry-without-diagnosis] ×3" in self._log(workspace)
|
||||
|
||||
def test_an_edited_patch_keeps_the_model_proposal(self, workspace, tmp_path):
|
||||
"""Overwriting `patch` lost the only record of what was proposed versus approved."""
|
||||
replacement = tmp_path / "new.txt"
|
||||
replacement.write_text("If the fetch fails, ask the user.", encoding="utf-8")
|
||||
exit_code = reflect_apply.main(
|
||||
["--id", "f7a2", "--new-text-file", str(replacement), "--workspace", str(workspace)]
|
||||
)
|
||||
|
||||
assert exit_code == 0
|
||||
record = _findings(workspace)[0]
|
||||
assert record["patch"]["new_text"] == "If the fetch fails, STOP and diagnose.", "the model's proposal"
|
||||
assert record["applied"]["new_text"] == "If the fetch fails, ask the user.", "what the user approved"
|
||||
assert record["applied"]["edited_by_user"] is True
|
||||
assert "ask the user" in (workspace / TARGET_REL).read_text(encoding="utf-8")
|
||||
assert "APPLIED-EDITED f7a2" in self._log(workspace)
|
||||
|
||||
def test_a_pre_migration_record_still_works(self, workspace):
|
||||
"""Findings decided before the reason existed carry a flat `rejected_at` — leave them be."""
|
||||
_write_findings(workspace, _record(id="fold", status="rejected", rejected_at="2026-09-01 10:55"), _record())
|
||||
assert reflect_apply.main(["--id", "f7a2", "--skip", "--workspace", str(workspace)]) == 0
|
||||
assert _findings(workspace)[0]["rejected_at"] == "2026-09-01 10:55"
|
||||
|
||||
def test_skip_refuses_a_finding_that_is_already_decided(self, workspace):
|
||||
_write_findings(workspace, _record(status="applied"))
|
||||
assert reflect_apply.main(["--id", "f7a2", "--skip", "--workspace", str(workspace)]) == 2
|
||||
|
||||
def test_skip_and_reject_are_mutually_exclusive(self, workspace):
|
||||
with pytest.raises(SystemExit):
|
||||
reflect_apply.main(["--id", "f7a2", "--skip", "--reject", "--reason", "x", "--workspace", str(workspace)])
|
||||
701
skills/reflect/tests/test_reflect_auto.py
Normal file
701
skills/reflect/tests/test_reflect_auto.py
Normal file
@@ -0,0 +1,701 @@
|
||||
"""Tests for reflect_auto.py — answer validation, batch persistence and the notification threshold."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
import reflect_auto
|
||||
|
||||
TODAY = "2026-09-08"
|
||||
WHEN = datetime(2026, 9, 8)
|
||||
|
||||
|
||||
def _stats(**overrides):
|
||||
"""The per-run statistics dict that feeds both the report and state.json."""
|
||||
stats = {"at": "2026-09-08 03:30", "sessions": 4, "batches": 1, "open": 0, "watch": 0, "repeat_per_100": 0.0}
|
||||
stats.update(overrides)
|
||||
return stats
|
||||
|
||||
|
||||
def _raw_finding(**overrides):
|
||||
finding = {
|
||||
"pattern": "retry-without-diagnosis",
|
||||
"severity": "medium",
|
||||
"diagnosis": "After an error the same call is repeated with identical arguments.",
|
||||
"evidence": [{"session": "websocket_abc", "when": "2026-07-11", "excerpt": "web_fetch → ERROR ×4"}],
|
||||
"occurrences": 1,
|
||||
"sessions_affected": 1,
|
||||
"proposal": "Add a hard STOP gate.",
|
||||
}
|
||||
finding.update(overrides)
|
||||
return finding
|
||||
|
||||
|
||||
def _answer(*findings) -> str:
|
||||
return "```json\n" + json.dumps({"findings": list(findings)}) + "\n```"
|
||||
|
||||
|
||||
def _parsed(answer: str, session_count: int = 10) -> list[dict]:
|
||||
"""Only the findings — notes about what was salvaged are asserted where they matter."""
|
||||
return reflect_auto.parse_findings(answer, session_count)[0]
|
||||
|
||||
|
||||
def _filed(**overrides):
|
||||
"""A record as it would already sit in findings.jsonl."""
|
||||
record = {
|
||||
"id": "f0001",
|
||||
"status": "watch",
|
||||
"created": "2026-09-01",
|
||||
"pattern": "retry-without-diagnosis",
|
||||
"severity": "medium",
|
||||
"diagnosis": "After an error the same call is repeated.",
|
||||
"evidence": [{"session": "websocket_old"}],
|
||||
"occurrences": 1,
|
||||
"sessions_affected": 1,
|
||||
"proposal": "Add a hard STOP gate.",
|
||||
}
|
||||
record.update(overrides)
|
||||
return record
|
||||
|
||||
|
||||
class TestParseFindings:
|
||||
def test_accepts_a_well_formed_answer(self):
|
||||
parsed = _parsed(_answer(_raw_finding()))
|
||||
assert parsed[0]["pattern"] == "retry-without-diagnosis"
|
||||
assert parsed[0]["patch"] is None
|
||||
|
||||
def test_accepts_an_empty_findings_list(self):
|
||||
assert _parsed(_answer()) == []
|
||||
|
||||
def test_accepts_bare_json_without_a_fence(self):
|
||||
assert _parsed(json.dumps({"findings": []})) == []
|
||||
|
||||
def test_ignores_narration_around_the_block(self):
|
||||
answer = "Here are the findings:\n" + _answer(_raw_finding()) + "\nDone."
|
||||
assert len(_parsed(answer)) == 1
|
||||
|
||||
def test_rejects_an_answer_without_json(self):
|
||||
with pytest.raises(reflect_auto.FindingsError):
|
||||
reflect_auto.parse_findings("I found three problems but I am not sending JSON.", 10)
|
||||
|
||||
def test_unescaped_quote_error_points_at_the_offending_text(self):
|
||||
"""The real failure seen on the first live run: a Czech „…" closing with ASCII ".
|
||||
|
||||
A bare "no parseable json" message gives the retry nothing to work with, so the
|
||||
error must name the line, the column and the surrounding text.
|
||||
"""
|
||||
broken = '```json\n{"findings": [{"pattern": "x", "diagnosis": "runtime said („blocked") and then"}]}\n```'
|
||||
with pytest.raises(reflect_auto.FindingsError) as raised:
|
||||
reflect_auto.parse_findings(broken, 10)
|
||||
message = str(raised.value)
|
||||
assert "line 1 column" in message
|
||||
assert "blocked" in message
|
||||
assert "unescaped double quote" in message
|
||||
|
||||
def test_error_is_located_inside_the_block_not_the_whole_reply(self):
|
||||
"""Column numbers measured across the fence would be meaningless to the model."""
|
||||
broken = 'Here is the result:\n\n```json\n{"findings": [{"diagnosis": "a "b" c"}]}\n```'
|
||||
with pytest.raises(reflect_auto.FindingsError) as raised:
|
||||
reflect_auto.parse_findings(broken, 10)
|
||||
assert "line 1 column" in str(raised.value)
|
||||
|
||||
def test_last_block_wins_when_the_model_shows_its_work(self):
|
||||
first = json.dumps({"findings": [_raw_finding(pattern="draft-version")]})
|
||||
second = json.dumps({"findings": [_raw_finding(pattern="final-version")]})
|
||||
answer = f"First draft:\n```json\n{first}\n```\nCorrected:\n```json\n{second}\n```"
|
||||
assert _parsed(answer)[0]["pattern"] == "final-version"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
{"pattern": "Retry Without Diagnosis"},
|
||||
{"pattern": "ab"},
|
||||
{"severity": "critical"},
|
||||
{"diagnosis": ""},
|
||||
{"occurrences": 0},
|
||||
{"occurrences": "sedm"},
|
||||
{"evidence": []},
|
||||
{"evidence": [{"when": "2026-07-11"}]},
|
||||
],
|
||||
)
|
||||
def test_malformed_finding_is_dropped_and_the_others_survive(self, override):
|
||||
"""Re-asking costs a ~420k token turn — one bad record must not throw the batch away."""
|
||||
parsed, problems = reflect_auto.parse_findings(
|
||||
_answer(_raw_finding(**override), _raw_finding(pattern="something-else")), 10
|
||||
)
|
||||
assert [finding["pattern"] for finding in parsed] == ["something-else"]
|
||||
assert any("dropped" in problem for problem in problems)
|
||||
|
||||
def test_an_answer_with_nothing_usable_still_raises(self):
|
||||
"""Nothing salvageable means the turn was wasted — that is worth one retry."""
|
||||
with pytest.raises(reflect_auto.FindingsError):
|
||||
_parsed(_answer(_raw_finding(severity="critical")))
|
||||
|
||||
def test_unknown_fields_are_ignored_not_fatal(self):
|
||||
parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(confidence=0.9)), 10)
|
||||
assert "confidence" not in parsed[0]
|
||||
assert any("confidence" in problem for problem in problems)
|
||||
|
||||
def test_extra_findings_are_trimmed_not_the_batch(self):
|
||||
many = [_raw_finding(pattern=f"pattern-{index}") for index in range(reflect_auto.MAX_FINDINGS_PER_BATCH + 1)]
|
||||
parsed, problems = reflect_auto.parse_findings(_answer(*many), 10)
|
||||
assert len(parsed) == reflect_auto.MAX_FINDINGS_PER_BATCH
|
||||
assert any("kept the first" in problem for problem in problems)
|
||||
|
||||
def test_overlong_diagnosis_is_truncated(self):
|
||||
parsed = _parsed(_answer(_raw_finding(diagnosis="x" * (reflect_auto.MAX_DIAGNOSIS_CHARS + 50))))
|
||||
assert len(parsed[0]["diagnosis"]) == reflect_auto.MAX_DIAGNOSIS_CHARS
|
||||
assert parsed[0]["diagnosis"].endswith("…")
|
||||
|
||||
|
||||
class TestParsePatch:
|
||||
def _patch(self, **overrides):
|
||||
patch = {"file": "skills/note/SKILL.md", "old_text": "try again", "new_text": "STOP and diagnose"}
|
||||
patch.update(overrides)
|
||||
return patch
|
||||
|
||||
def test_accepts_a_complete_patch(self):
|
||||
parsed = _parsed(_answer(_raw_finding(patch=self._patch())))
|
||||
assert parsed[0]["patch"]["file"] == "skills/note/SKILL.md"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
{"old_text": ""},
|
||||
{"new_text": "try again"},
|
||||
{"file": "/etc/passwd"},
|
||||
{"file": "../../../etc/passwd"},
|
||||
],
|
||||
)
|
||||
def test_unsafe_or_empty_patch_is_dropped_but_the_finding_survives(self, override):
|
||||
"""The diagnosis and the proposal are still worth reviewing without a patch."""
|
||||
parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(patch=self._patch(**override))), 10)
|
||||
assert len(parsed) == 1
|
||||
assert parsed[0]["patch"] is None
|
||||
assert any("kept the finding without it" in problem for problem in problems)
|
||||
|
||||
def test_incomplete_patch_is_dropped_but_the_finding_survives(self):
|
||||
parsed = _parsed(_answer(_raw_finding(patch={"file": "a.md", "old_text": "x"})))
|
||||
assert parsed[0]["patch"] is None
|
||||
|
||||
|
||||
class TestCounts:
|
||||
"""The counts drive the threshold and the ranking, and nothing but this checks them."""
|
||||
|
||||
def test_more_sessions_than_occurrences_is_impossible(self):
|
||||
"""Seen live: a filed finding claimed 4 occurrences across 5 sessions."""
|
||||
parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(occurrences=4, sessions_affected=5)), 10)
|
||||
assert parsed[0]["sessions_affected"] == 4
|
||||
assert any("clamped" in problem for problem in problems)
|
||||
|
||||
def test_more_sessions_than_the_slice_held_is_impossible(self):
|
||||
parsed, _ = reflect_auto.parse_findings(_answer(_raw_finding(occurrences=9, sessions_affected=6)), 3)
|
||||
assert parsed[0]["sessions_affected"] == 3
|
||||
|
||||
def test_a_coherent_count_is_left_alone(self):
|
||||
parsed, problems = reflect_auto.parse_findings(_answer(_raw_finding(occurrences=7, sessions_affected=4)), 10)
|
||||
assert (parsed[0]["occurrences"], parsed[0]["sessions_affected"]) == (7, 4)
|
||||
assert problems == []
|
||||
|
||||
|
||||
class TestEvidenceFolding:
|
||||
"""`occurrences` sums across runs, so the examples behind it have to survive the fold."""
|
||||
|
||||
def _evidence(self, session: str) -> list[dict]:
|
||||
return [{"session": session, "when": "2026-09-02", "excerpt": "web_fetch → ERROR"}]
|
||||
|
||||
def test_the_fold_keeps_the_older_evidence(self):
|
||||
existing = [_filed(evidence=self._evidence("websocket_old"))]
|
||||
raw = _raw_finding(evidence=self._evidence("websocket_new"))
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY)
|
||||
|
||||
sessions = [item["session"] for item in merged[0].evidence]
|
||||
assert sessions == ["websocket_new", "websocket_old"], "newest first, older behind it"
|
||||
|
||||
def test_the_same_evidence_twice_is_kept_once(self):
|
||||
existing = [_filed(evidence=self._evidence("websocket_same"))]
|
||||
raw = _raw_finding(evidence=self._evidence("websocket_same"))
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY)
|
||||
|
||||
assert len(merged[0].evidence) == 1
|
||||
|
||||
def test_the_evidence_list_is_capped(self):
|
||||
older = [{"session": f"websocket_{i}", "excerpt": str(i)} for i in range(10)]
|
||||
raw = _raw_finding(evidence=self._evidence("websocket_new"))
|
||||
merged = reflect_auto.merge_findings([_filed(evidence=older)], _parsed(_answer(raw)), TODAY)
|
||||
|
||||
assert len(merged[0].evidence) == reflect_auto.MAX_EVIDENCE
|
||||
|
||||
def test_a_regression_does_not_mix_evidence_from_before_the_fix(self):
|
||||
existing = [_filed(status="applied", evidence=self._evidence("websocket_old"))]
|
||||
raw = _raw_finding(evidence=self._evidence("websocket_new"))
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY)
|
||||
|
||||
assert [item["session"] for item in merged[0].evidence] == ["websocket_new"]
|
||||
|
||||
|
||||
class TestThreshold:
|
||||
def test_first_sighting_stays_silent(self):
|
||||
"""A single occurrence is noise, not a pattern — it must not reach Telegram."""
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(_raw_finding())), TODAY)
|
||||
assert merged[0].status == reflect_auto.STATUS_WATCH
|
||||
|
||||
def test_repeat_within_one_batch_opens_immediately(self):
|
||||
raw = _raw_finding(occurrences=7, sessions_affected=4)
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)
|
||||
assert merged[0].status == reflect_auto.STATUS_OPEN
|
||||
|
||||
def test_many_occurrences_in_one_session_stay_silent(self):
|
||||
"""One session looping seven times is still one session — not yet a habit."""
|
||||
raw = _raw_finding(occurrences=7, sessions_affected=1)
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)
|
||||
assert merged[0].status == reflect_auto.STATUS_WATCH
|
||||
|
||||
def test_second_sighting_promotes_a_watched_pattern(self):
|
||||
merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding())), TODAY)
|
||||
assert merged[0].status == reflect_auto.STATUS_OPEN
|
||||
assert merged[0].occurrences == 2
|
||||
assert merged[0].sessions_affected == 2
|
||||
|
||||
def test_promotion_records_the_superseded_record(self):
|
||||
merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding())), TODAY)
|
||||
assert merged[0].history == ("2026-09-01:f0001",)
|
||||
|
||||
def test_reappearing_after_apply_is_a_regression(self):
|
||||
"""A fixed pattern coming back must open at once, however few the occurrences."""
|
||||
merged = reflect_auto.merge_findings([_filed(status="applied")], _parsed(_answer(_raw_finding())), TODAY)
|
||||
assert merged[0].status == reflect_auto.STATUS_OPEN
|
||||
assert merged[0].regression_of == "f0001"
|
||||
assert merged[0].occurrences == 1
|
||||
|
||||
def test_evidence_from_before_the_fix_is_not_a_regression(self):
|
||||
"""The window reaches back over sessions that predate the fix; they say nothing about it."""
|
||||
applied = _filed(status="applied", applied={"at": "2026-09-01 10:00", "sha": "abc", "file": "SOUL.md"})
|
||||
raw = _raw_finding(occurrences=9, sessions_affected=5, evidence=[{"session": "ws", "when": "2026-08-31"}])
|
||||
merged = reflect_auto.merge_findings([applied], _parsed(_answer(raw)), TODAY)
|
||||
|
||||
assert merged[0].regression_of is None
|
||||
assert merged[0].status == reflect_auto.STATUS_WATCH, "a finding about the past must not open"
|
||||
|
||||
def test_evidence_from_after_the_fix_is_a_regression(self):
|
||||
applied = _filed(status="applied", applied={"at": "2026-09-01 10:00", "sha": "abc", "file": "SOUL.md"})
|
||||
raw = _raw_finding(evidence=[{"session": "ws", "when": "2026-09-02 13:53"}])
|
||||
merged = reflect_auto.merge_findings([applied], _parsed(_answer(raw)), TODAY)
|
||||
|
||||
assert merged[0].regression_of == "f0001"
|
||||
assert merged[0].status == reflect_auto.STATUS_OPEN
|
||||
|
||||
def test_rejected_pattern_never_opens_again(self):
|
||||
"""Rejection is a decision, not a deferral — a repeat must not start nagging again."""
|
||||
raw = _raw_finding(occurrences=9, sessions_affected=5)
|
||||
merged = reflect_auto.merge_findings([_filed(status="rejected")], _parsed(_answer(raw)), TODAY)
|
||||
assert merged[0].status == reflect_auto.STATUS_WATCH
|
||||
|
||||
def test_rejection_survives_the_run_after_next(self):
|
||||
"""The watch record filed after a rejection is newer — it must not supersede the decision."""
|
||||
store = [_filed(status="rejected")]
|
||||
raw = _raw_finding(occurrences=9, sessions_affected=5)
|
||||
|
||||
first = reflect_auto.merge_findings(store, _parsed(_answer(raw)), "2026-09-02")
|
||||
second = reflect_auto.merge_findings(reflect_auto.supersede(store, first), _parsed(_answer(raw)), "2026-09-03")
|
||||
assert second[0].status == reflect_auto.STATUS_WATCH
|
||||
|
||||
def test_rejected_pattern_does_not_inherit_counts(self):
|
||||
merged = reflect_auto.merge_findings([_filed(status="rejected")], _parsed(_answer(_raw_finding())), TODAY)
|
||||
assert merged[0].occurrences == 1
|
||||
|
||||
def test_regression_flag_survives_a_second_sighting(self):
|
||||
"""A pattern that came back after a fix stays flagged until it is dealt with."""
|
||||
applied = [_filed(status="applied")]
|
||||
first = reflect_auto.merge_findings(applied, _parsed(_answer(_raw_finding())), "2026-09-02")
|
||||
second = reflect_auto.merge_findings(
|
||||
reflect_auto.supersede(applied, first), _parsed(_answer(_raw_finding())), "2026-09-03"
|
||||
)
|
||||
assert second[0].regression_of == "f0001"
|
||||
|
||||
def test_ids_are_unique_against_existing_records(self):
|
||||
merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding(pattern="other-thing"))), TODAY)
|
||||
assert merged[0].id != "f0001"
|
||||
|
||||
|
||||
class TestLastSeen:
|
||||
"""`created` is when a record was filed; only the evidence says when the pattern last occurred."""
|
||||
|
||||
def _merge(self, *evidence: dict) -> reflect_auto.Finding:
|
||||
raw = _raw_finding(evidence=list(evidence))
|
||||
return reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)[0]
|
||||
|
||||
def test_the_newest_evidence_date_wins(self):
|
||||
finding = self._merge(
|
||||
{"session": "ws_a", "when": "2026-08-20"},
|
||||
{"session": "ws_b", "when": "2026-08-31 13:53"},
|
||||
{"session": "ws_c", "when": "2026-08-25"},
|
||||
)
|
||||
assert finding.last_seen == "2026-08-31", "mixed shapes, only the leading date counts"
|
||||
|
||||
def test_undated_evidence_falls_back_to_created(self):
|
||||
finding = self._merge({"session": "ws_a", "excerpt": "no when at all"})
|
||||
assert finding.last_seen == TODAY
|
||||
|
||||
def test_non_date_when_is_ignored(self):
|
||||
finding = self._merge({"session": "ws_a", "when": "yesterday"}, {"session": "ws_b", "when": "turn 4"})
|
||||
assert finding.last_seen == TODAY
|
||||
|
||||
def test_folded_evidence_from_an_earlier_run_counts_too(self):
|
||||
existing = [_filed(evidence=[{"session": "ws_old", "when": "2026-09-05"}])]
|
||||
raw = _raw_finding(evidence=[{"session": "ws_new", "when": "2026-08-01"}])
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(raw)), TODAY)
|
||||
assert merged[0].last_seen == "2026-09-05"
|
||||
|
||||
def test_last_seen_is_stored_for_the_review_to_sort_by(self):
|
||||
assert self._merge({"session": "ws_a", "when": "2026-08-20"}).to_json()["last_seen"] == "2026-08-20"
|
||||
|
||||
|
||||
class TestSupersede:
|
||||
def test_folded_record_is_replaced_not_duplicated(self):
|
||||
existing = [_filed()]
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(_raw_finding())), TODAY)
|
||||
result = reflect_auto.supersede(existing, merged)
|
||||
assert len(result) == 1
|
||||
assert result[0]["status"] == reflect_auto.STATUS_OPEN
|
||||
|
||||
def test_unrelated_records_survive(self):
|
||||
existing = [_filed(id="f0001", pattern="other-thing", status="applied")]
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(_raw_finding())), TODAY)
|
||||
result = reflect_auto.supersede(existing, merged)
|
||||
assert {record["pattern"] for record in result} == {"other-thing", "retry-without-diagnosis"}
|
||||
|
||||
def test_applied_history_is_kept_for_regression_tracking(self):
|
||||
existing = [_filed(status="applied")]
|
||||
merged = reflect_auto.merge_findings(existing, _parsed(_answer(_raw_finding())), TODAY)
|
||||
result = reflect_auto.supersede(existing, merged)
|
||||
assert len(result) == 2
|
||||
|
||||
|
||||
class TestRendering:
|
||||
def test_known_patterns_feed_the_id_vocabulary_back(self):
|
||||
text = reflect_auto._known_patterns([_filed()])
|
||||
assert "`retry-without-diagnosis`" in text
|
||||
assert "[watch]" in text
|
||||
|
||||
def test_known_patterns_handles_an_empty_store(self):
|
||||
assert "none yet" in reflect_auto._known_patterns([])
|
||||
|
||||
def test_report_marks_regressions(self):
|
||||
merged = reflect_auto.merge_findings([_filed(status="applied")], _parsed(_answer(_raw_finding())), TODAY)
|
||||
report = reflect_auto.render_report(merged, _stats(), None, WHEN)
|
||||
assert "REGRESSION" in report
|
||||
|
||||
def test_report_renders_a_patch_as_a_diff(self):
|
||||
raw = _raw_finding(patch={"file": "a.md", "old_text": "try again", "new_text": "STOP"})
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)
|
||||
report = reflect_auto.render_report(merged, _stats(), None, WHEN)
|
||||
assert "```diff" in report
|
||||
assert "- try again" in report
|
||||
assert "+ STOP" in report
|
||||
|
||||
def test_report_shows_how_old_a_finding_is(self):
|
||||
"""`occurrences` is cumulative, so the count alone cannot say whether this is still live."""
|
||||
merged = reflect_auto.merge_findings([_filed()], _parsed(_answer(_raw_finding())), TODAY)
|
||||
report = reflect_auto.render_report(merged, _stats(), None, WHEN)
|
||||
assert "first seen 2026-09-01, last seen 2026-07-11" in report, "the evidence date, not the refile date"
|
||||
|
||||
def test_a_brand_new_finding_reports_one_date(self):
|
||||
raw = _raw_finding(evidence=[{"session": "websocket_abc", "excerpt": "no date here"}])
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)
|
||||
report = reflect_auto.render_report(merged, _stats(), None, WHEN)
|
||||
assert f"first seen {TODAY}, last seen {TODAY}" in report
|
||||
|
||||
def test_report_marks_a_finding_the_window_no_longer_reaches(self):
|
||||
"""Nothing in the window backs it any more, so no future run will refresh it either."""
|
||||
raw = _raw_finding(occurrences=7, sessions_affected=4)
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)
|
||||
report = reflect_auto.render_report(merged, _stats(window_from="2026-08-18T00:00:00"), None, WHEN)
|
||||
assert "STALE" in report
|
||||
|
||||
def test_a_finding_inside_the_window_is_not_stale(self):
|
||||
raw = _raw_finding(evidence=[{"session": "websocket_abc", "when": "2026-09-01", "excerpt": "x"}])
|
||||
merged = reflect_auto.merge_findings([], _parsed(_answer(raw)), TODAY)
|
||||
report = reflect_auto.render_report(merged, _stats(window_from="2026-08-18T00:00:00"), None, WHEN)
|
||||
assert "STALE" not in report
|
||||
|
||||
def test_empty_report_says_so(self):
|
||||
report = reflect_auto.render_report([], _stats(), None, WHEN)
|
||||
assert "Nothing to report" in report
|
||||
|
||||
def test_report_shows_the_known_pattern_rate(self):
|
||||
"""The one number proving the skill pays off — without it results/ is dead weight."""
|
||||
report = reflect_auto.render_report([], _stats(repeat_per_100=3.2), 5.1, WHEN)
|
||||
assert "Known patterns: 3.2 occurrences / 100 sessions (previous run 5.1)." in report
|
||||
|
||||
def test_first_run_has_no_trend_to_show(self):
|
||||
assert "(minule" not in reflect_auto.render_report([], _stats(), None, WHEN)
|
||||
|
||||
|
||||
class TestPromptContract:
|
||||
def test_prompt_forbids_writing(self):
|
||||
"""The read-only instruction is one of the two guards on the analysis run."""
|
||||
prompt = reflect_auto.build_prompt("digest", "none", 3)
|
||||
assert "Write nothing" in prompt
|
||||
|
||||
def test_prompt_carries_digest_and_known_patterns(self):
|
||||
prompt = reflect_auto.build_prompt("SESSION-DIGEST-HERE", "PATTERN-LIST-HERE", 3)
|
||||
assert "SESSION-DIGEST-HERE" in prompt
|
||||
assert "PATTERN-LIST-HERE" in prompt
|
||||
|
||||
def test_prompt_caps_file_reads(self):
|
||||
"""Every extra tool iteration re-prefills the whole digest — the cap is what keeps a batch cheap."""
|
||||
assert "Read at most 2 files" in reflect_auto.build_prompt("digest", "none", 3)
|
||||
|
||||
|
||||
class FakeBot:
|
||||
"""Stands in for Nanobot: hands out canned replies and records what it was asked."""
|
||||
|
||||
def __init__(self, replies: list[Any]):
|
||||
self._replies = list(replies)
|
||||
self.calls: list[tuple[str, str]] = []
|
||||
|
||||
async def run(self, message: str, *, session_key: str):
|
||||
self.calls.append((message, session_key))
|
||||
reply = self._replies.pop(0) if len(self._replies) > 1 else self._replies[0]
|
||||
if isinstance(reply, Exception):
|
||||
raise reply
|
||||
return reply
|
||||
|
||||
|
||||
def _reply(content: str = "", stop_reason: str | None = None, error: str | None = None) -> SimpleNamespace:
|
||||
return SimpleNamespace(content=content, stop_reason=stop_reason, error=error)
|
||||
|
||||
|
||||
def _llm_failure() -> SimpleNamespace:
|
||||
"""What nanobot hands back when the provider gives up: the error text as the reply."""
|
||||
return _reply("Error calling LLM: timed out after 300s", stop_reason="error", error="timed out after 300s")
|
||||
|
||||
|
||||
def _session_records(started: str, turns: int = 3) -> list[dict]:
|
||||
"""A conversation long enough to clear reflect_distill.MIN_MESSAGES."""
|
||||
records: list[dict] = [{"_type": "metadata", "key": "websocket:x", "created_at": started}]
|
||||
for i in range(turns):
|
||||
records.append({"role": "user", "content": f"dotaz {i}"})
|
||||
records.append({"role": "assistant", "content": f"answer {i}"})
|
||||
return records
|
||||
|
||||
|
||||
EARLIER = "2026-07-11T14:02:03"
|
||||
LATER = "2026-08-20T09:15:00"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path):
|
||||
"""A workspace that is a real git repo with two analysable sessions, like the server's."""
|
||||
sessions = tmp_path / "sessions"
|
||||
sessions.mkdir()
|
||||
for name, started in (("websocket_first", EARLIER), ("websocket_second", LATER)):
|
||||
records = _session_records(started)
|
||||
(sessions / f"{name}.jsonl").write_text(
|
||||
"\n".join(json.dumps(r, ensure_ascii=False) for r in records), encoding="utf-8"
|
||||
)
|
||||
(tmp_path / "reflect").mkdir()
|
||||
|
||||
for args in (
|
||||
["init", "-q"],
|
||||
["config", "user.email", "test@example.com"],
|
||||
["config", "user.name", "test"],
|
||||
["add", "-A"],
|
||||
["commit", "-q", "-m", "init"],
|
||||
):
|
||||
subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _args(**overrides):
|
||||
"""One session per batch (budget_chars=1), so the two fixture sessions make two batches."""
|
||||
args = {
|
||||
"dry_run": False,
|
||||
"all": False,
|
||||
"budget_chars": 1,
|
||||
"max_batches": 0,
|
||||
"deadline_minutes": 20,
|
||||
"window_days": 0,
|
||||
}
|
||||
args.update(overrides)
|
||||
return SimpleNamespace(**args)
|
||||
|
||||
|
||||
def _state(workspace: Path) -> dict:
|
||||
return json.loads((workspace / "reflect" / "state.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _findings(workspace: Path) -> list[dict]:
|
||||
path = workspace / "reflect" / "findings.jsonl"
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _seed_findings(workspace: Path, *records: dict) -> None:
|
||||
path = workspace / "reflect" / "findings.jsonl"
|
||||
path.write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in records) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _run(workspace: Path, bot, monkeypatch, **arg_overrides) -> tuple[str, int]:
|
||||
monkeypatch.setattr(reflect_auto, "_open_bot", lambda preset: bot)
|
||||
return asyncio.run(reflect_auto._run(workspace, _args(**arg_overrides), WHEN))
|
||||
|
||||
|
||||
class TestRunPersistence:
|
||||
"""The 2026-09-02 failure: a 30min timeout threw away two finished batches and kept the cursor."""
|
||||
|
||||
def test_every_batch_is_filed_as_it_finishes(self, workspace, monkeypatch):
|
||||
bot = FakeBot(
|
||||
[
|
||||
_reply(_answer(_raw_finding())),
|
||||
_reply(_answer(_raw_finding(pattern="tool-call-leaked-as-text"))),
|
||||
]
|
||||
)
|
||||
_run(workspace, bot, monkeypatch)
|
||||
|
||||
assert {record["pattern"] for record in _findings(workspace)} == {
|
||||
"retry-without-diagnosis",
|
||||
"tool-call-leaked-as-text",
|
||||
}
|
||||
assert _state(workspace)["cursor"] == LATER
|
||||
assert [run["batches"] for run in _state(workspace)["runs"]] == [2], "one run entry, updated in place"
|
||||
|
||||
def test_the_same_pattern_in_two_batches_folds_into_one_record(self, workspace, monkeypatch):
|
||||
"""Per-batch filing must not turn cross-batch dedup into duplicate records."""
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
_run(workspace, bot, monkeypatch)
|
||||
|
||||
records = _findings(workspace)
|
||||
assert len(records) == 1
|
||||
assert records[0]["occurrences"] == 2
|
||||
assert records[0]["status"] == "open", "the second sighting is what promotes a watched pattern"
|
||||
|
||||
def test_the_report_counts_a_repeated_pattern_once(self, workspace, monkeypatch):
|
||||
"""The report and the Telegram line have to say what the store holds, not how often it was refiled."""
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
message, _ = _run(workspace, bot, monkeypatch)
|
||||
|
||||
report = (workspace / "results" / f"{WHEN:%Y-%m-%d}_reflect.md").read_text(encoding="utf-8")
|
||||
assert len([line for line in report.splitlines() if line.startswith("## ")]) == 1
|
||||
assert "Findings: 1 (1 to review, 0 watched)" in report
|
||||
assert _state(workspace)["runs"][-1]["open"] == 1
|
||||
assert "1 findings to review" in message
|
||||
|
||||
def test_a_batch_that_dies_does_not_take_the_finished_one_with_it(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_reply(_answer(_raw_finding())), RuntimeError("provider down")])
|
||||
with pytest.raises(RuntimeError):
|
||||
_run(workspace, bot, monkeypatch)
|
||||
|
||||
assert len(_findings(workspace)) == 1
|
||||
assert _state(workspace)["cursor"] == EARLIER
|
||||
assert _state(workspace)["runs"][-1]["batches"] == 1
|
||||
|
||||
def test_the_cursor_never_moves_backwards(self, workspace, monkeypatch):
|
||||
"""An `--all` walk starts at the oldest session and must not undo the nightly progress."""
|
||||
reflect_auto._save_state(workspace, {"cursor": "2026-12-31T00:00:00", "runs": []})
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
_run(workspace, bot, monkeypatch, all=True)
|
||||
|
||||
assert _state(workspace)["cursor"] == "2026-12-31T00:00:00"
|
||||
|
||||
def test_the_deadline_stops_the_run_and_says_so(self, workspace, monkeypatch):
|
||||
_seed_findings(workspace, _filed())
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
message, _ = _run(workspace, bot, monkeypatch, deadline_minutes=0)
|
||||
|
||||
assert len(bot.calls) == 1
|
||||
assert _state(workspace)["cursor"] == EARLIER
|
||||
assert "Analysed 1/2 batches." in message
|
||||
|
||||
def test_a_finished_run_reports_no_partial_progress(self, workspace, monkeypatch):
|
||||
_seed_findings(workspace, _filed())
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
message, _ = _run(workspace, bot, monkeypatch)
|
||||
|
||||
assert "batches" not in message
|
||||
|
||||
def test_an_unfinished_run_speaks_up_even_with_no_findings(self, workspace, monkeypatch):
|
||||
"""Silence here is how a starved cursor went unnoticed for three nights."""
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
message, _ = _run(workspace, bot, monkeypatch, deadline_minutes=0)
|
||||
|
||||
assert "1 batches left" in message
|
||||
|
||||
def test_a_finished_run_with_no_findings_stays_quiet(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_reply(_answer())])
|
||||
message, _ = _run(workspace, bot, monkeypatch)
|
||||
|
||||
assert message == ""
|
||||
|
||||
|
||||
class TestWindow:
|
||||
"""Findings must describe recent behaviour; a months-old backlog must not starve the run."""
|
||||
|
||||
def test_the_window_skips_everything_older(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
_run(workspace, bot, monkeypatch, window_days=21)
|
||||
|
||||
assert len(bot.calls) == 1, "only the session inside the window is analysable"
|
||||
assert "websocket_second" in bot.calls[0][0]
|
||||
assert "websocket_first" not in bot.calls[0][0]
|
||||
assert _state(workspace)["cursor"] == LATER, "the skipped backlog is skipped for good"
|
||||
|
||||
def test_the_cursor_still_wins_over_the_window(self, workspace, monkeypatch):
|
||||
"""The window is a ceiling, not a rewind — nothing already counted may be re-read."""
|
||||
reflect_auto._save_state(workspace, {"cursor": LATER, "runs": []})
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
message, sessions = _run(workspace, bot, monkeypatch, window_days=365)
|
||||
|
||||
assert bot.calls == []
|
||||
assert (message, sessions) == ("", 0)
|
||||
|
||||
def test_the_report_says_what_the_window_covered(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_reply(_answer(_raw_finding()))])
|
||||
_run(workspace, bot, monkeypatch, window_days=21)
|
||||
|
||||
report = (workspace / "results" / f"{WHEN:%Y-%m-%d}_reflect.md").read_text(encoding="utf-8")
|
||||
assert "Window: from 2026-08-18, batches 1/1." in report
|
||||
|
||||
def test_a_live_all_run_is_refused(self, workspace):
|
||||
"""`--all` re-reads counted sessions and merge_findings would sum their occurrences in."""
|
||||
with pytest.raises(SystemExit):
|
||||
reflect_auto.main(["--all", "--workspace", str(workspace)])
|
||||
|
||||
def test_a_dry_all_run_is_allowed(self, workspace):
|
||||
assert reflect_auto.main(["--all", "--dry-run", "--workspace", str(workspace)]) == 0
|
||||
assert list((workspace / "tmp").glob("reflect-batch.*.md"))
|
||||
|
||||
|
||||
class TestLlmErrorHandling:
|
||||
"""A dead provider is not a malformed answer; conflating the two burned a validator attempt."""
|
||||
|
||||
def test_a_provider_failure_re_asks_the_original_question(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_llm_failure(), _reply(_answer(_raw_finding()))])
|
||||
parsed = asyncio.run(reflect_auto._resolve_findings(bot, "reflect:test-0", "PROMPT", workspace, 4))
|
||||
|
||||
assert len(parsed) == 1
|
||||
first, second = bot.calls
|
||||
assert second[0] == "PROMPT", "the retry must re-ask, not blame the model for the error text"
|
||||
assert second[1] != first[1], "the failed turn poisoned that session; retry needs a fresh key"
|
||||
|
||||
def test_a_dead_provider_is_reported_as_such(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_llm_failure()])
|
||||
with pytest.raises(reflect_auto.ReflectError, match="model unavailable"):
|
||||
asyncio.run(reflect_auto._resolve_findings(bot, "reflect:test-0", "PROMPT", workspace, 4))
|
||||
|
||||
assert len(bot.calls) == reflect_auto.MAX_LLM_ERROR_RETRIES + 1
|
||||
|
||||
def test_a_malformed_answer_still_gets_validator_feedback(self, workspace, monkeypatch):
|
||||
bot = FakeBot([_reply("no json here"), _reply(_answer(_raw_finding()))])
|
||||
parsed = asyncio.run(reflect_auto._resolve_findings(bot, "reflect:test-0", "PROMPT", workspace, 4))
|
||||
|
||||
assert len(parsed) == 1
|
||||
assert "rejected by the validator" in bot.calls[1][0]
|
||||
241
skills/reflect/tests/test_reflect_distill.py
Normal file
241
skills/reflect/tests/test_reflect_distill.py
Normal file
@@ -0,0 +1,241 @@
|
||||
"""Tests for reflect_distill.py — mechanical distillation of session logs."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
import reflect_distill
|
||||
|
||||
|
||||
def _write_session(directory: Path, name: str, records: list[dict]) -> Path:
|
||||
path = directory / f"{name}.jsonl"
|
||||
path.write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in records), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _conversation(turns: int = 3) -> list[dict]:
|
||||
"""A session long enough to clear MIN_MESSAGES."""
|
||||
records: list[dict] = [{"_type": "metadata", "key": "websocket:x", "created_at": "2026-07-11T14:02:03"}]
|
||||
for i in range(turns):
|
||||
records.append({"role": "user", "content": f"dotaz {i}"})
|
||||
records.append({"role": "assistant", "content": f"answer {i}"})
|
||||
return records
|
||||
|
||||
|
||||
class TestNoiseFilter:
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"reflect_20260908-033000",
|
||||
"compact-memory-auto_20260715-054147",
|
||||
"detach_abc",
|
||||
"dream_xyz",
|
||||
"cron_e81cda77",
|
||||
"cli_kimi-ollama-test",
|
||||
"wiki-compile",
|
||||
"note-compile",
|
||||
],
|
||||
)
|
||||
def test_machinery_prefixes_are_noise(self, name):
|
||||
assert reflect_distill.is_noise(name)
|
||||
|
||||
@pytest.mark.parametrize("name", ["wiki-capture-test2", "test-note-search-zzz"])
|
||||
def test_prefix_variants_are_noise(self, name):
|
||||
"""Exact matching used to let `…test2` through; the filter matches prefixes."""
|
||||
assert reflect_distill.is_noise(name)
|
||||
|
||||
@pytest.mark.parametrize("name", ["websocket_e5a6aac3-2c0a", "telegram_8826147089"])
|
||||
def test_real_conversations_are_kept(self, name):
|
||||
assert not reflect_distill.is_noise(name)
|
||||
|
||||
def test_base64_session_names_are_decoded(self):
|
||||
"""Older sessions are stored base64-encoded; `ZHJlYW06…` is `dream:…`."""
|
||||
assert reflect_distill.is_noise("ZHJlYW06MjAyNjA4MzEtMTA0NjM2")
|
||||
assert not reflect_distill.is_noise("d2Vic29ja2V0OjUyZDBmMzM4")
|
||||
|
||||
def test_reflect_excludes_its_own_sessions(self):
|
||||
"""Without this the skill would analyse its own runs on the next pass."""
|
||||
assert reflect_distill.is_noise("reflect_20260908-033000")
|
||||
|
||||
|
||||
class TestDistillSession:
|
||||
def test_short_sessions_are_skipped(self, tmp_path):
|
||||
path = _write_session(tmp_path, "websocket_short", [{"role": "user", "content": "ahoj"}])
|
||||
assert reflect_distill.distill_session(path) is None
|
||||
|
||||
def test_noise_sessions_are_skipped(self, tmp_path):
|
||||
path = _write_session(tmp_path, "dream_nightly", _conversation())
|
||||
assert reflect_distill.distill_session(path) is None
|
||||
|
||||
def test_header_carries_key_time_and_count(self, tmp_path):
|
||||
path = _write_session(tmp_path, "websocket_abc", _conversation())
|
||||
digest = reflect_distill.distill_session(path)
|
||||
assert digest.text.startswith(reflect_distill.SESSION_MARKER)
|
||||
assert "websocket_abc" in digest.text
|
||||
assert "2026-07-11 14:02" in digest.text
|
||||
assert digest.message_count == 6
|
||||
|
||||
def test_header_is_not_a_markdown_heading(self, tmp_path):
|
||||
"""Assistant prose is full of `###`, so a heading would not read as a boundary."""
|
||||
records = _conversation()
|
||||
records.append({"role": "assistant", "content": "### Summary\ndone"})
|
||||
path = _write_session(tmp_path, "websocket_abc", records)
|
||||
digest = reflect_distill.distill_session(path)
|
||||
assert not digest.text.startswith("#")
|
||||
assert digest.text.count(reflect_distill.SESSION_MARKER) == 1
|
||||
|
||||
def test_user_and_assistant_prose_is_kept_whole(self, tmp_path):
|
||||
prose = "this is a long answer " * 40
|
||||
records = _conversation()
|
||||
records.append({"role": "assistant", "content": prose})
|
||||
path = _write_session(tmp_path, "websocket_abc", records)
|
||||
assert prose.strip() in reflect_distill.distill_session(path).text
|
||||
|
||||
def test_slash_commands_are_dropped(self, tmp_path):
|
||||
records = _conversation()
|
||||
records.append({"role": "user", "content": "/model", "_command": True})
|
||||
path = _write_session(tmp_path, "websocket_abc", records)
|
||||
assert "/model" not in reflect_distill.distill_session(path).text
|
||||
|
||||
def test_decoded_key_is_used_as_name(self, tmp_path):
|
||||
path = _write_session(tmp_path, "d2Vic29ja2V0OjUyZDBmMzM4", _conversation())
|
||||
assert reflect_distill.distill_session(path).name == "websocket:52d0f338"
|
||||
|
||||
|
||||
class TestToolRendering:
|
||||
def _digest_with_tool(self, tmp_path, call, result):
|
||||
records = _conversation()
|
||||
records.append({"role": "assistant", "content": "", "tool_calls": [call]})
|
||||
records.append(result)
|
||||
path = _write_session(tmp_path, "websocket_abc", records)
|
||||
return reflect_distill.distill_session(path).text
|
||||
|
||||
def test_tool_result_body_is_replaced_by_metadata(self, tmp_path):
|
||||
"""Tool results are 89% of the corpus and carry almost no diagnostic value."""
|
||||
body = "x" * 5000
|
||||
text = self._digest_with_tool(
|
||||
tmp_path,
|
||||
{"id": "c1", "function": {"name": "read_file", "arguments": '{"path": "/tmp/a.md"}'}},
|
||||
{"role": "tool", "tool_call_id": "c1", "name": "read_file", "content": body},
|
||||
)
|
||||
assert body not in text
|
||||
assert "read_file: ok, 5.0 kB" in text
|
||||
|
||||
def test_errors_are_flagged_with_their_message(self, tmp_path):
|
||||
text = self._digest_with_tool(
|
||||
tmp_path,
|
||||
{"id": "c1", "function": {"name": "web_fetch", "arguments": '{"url": "https://x.dev"}'}},
|
||||
{"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "Error: 403 Forbidden"},
|
||||
)
|
||||
assert "ERROR Error: 403 Forbidden" in text
|
||||
|
||||
def test_workspace_path_prefix_is_stripped(self, tmp_path):
|
||||
text = self._digest_with_tool(
|
||||
tmp_path,
|
||||
{
|
||||
"id": "c1",
|
||||
"function": {
|
||||
"name": "read_file",
|
||||
"arguments": '{"path": "/home/nanobot/.nanobot/workspace/skills/note/SKILL.md"}',
|
||||
},
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "c1", "name": "read_file", "content": "ok"},
|
||||
)
|
||||
assert "path=skills/note/SKILL.md" in text
|
||||
|
||||
def test_malformed_arguments_still_render(self, tmp_path):
|
||||
"""A truncated argument blob is itself a signal worth seeing, not a crash."""
|
||||
text = self._digest_with_tool(
|
||||
tmp_path,
|
||||
{"id": "c1", "function": {"name": "exec", "arguments": '{"command": "ls'}},
|
||||
{"role": "tool", "tool_call_id": "c1", "name": "exec", "content": "ok"},
|
||||
)
|
||||
assert "exec(" in text
|
||||
|
||||
def test_long_urls_keep_their_distinguishing_tail(self, tmp_path):
|
||||
"""Front-truncation made different fetches look identical and hid real retries."""
|
||||
base = "https://raw.githubusercontent.com/some-owner/some-repo/main/packages/core/src/"
|
||||
first = self._digest_with_tool(
|
||||
tmp_path,
|
||||
{"id": "c1", "function": {"name": "web_fetch", "arguments": json.dumps({"url": base + "alpha.ts"})}},
|
||||
{"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "ok"},
|
||||
)
|
||||
second = self._digest_with_tool(
|
||||
tmp_path,
|
||||
{"id": "c1", "function": {"name": "web_fetch", "arguments": json.dumps({"url": base + "omega.ts"})}},
|
||||
{"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "ok"},
|
||||
)
|
||||
assert "alpha.ts" in first
|
||||
assert "omega.ts" in second
|
||||
assert first != second
|
||||
|
||||
def test_repeated_identical_calls_render_identically(self, tmp_path):
|
||||
"""The LLM detects retry loops by seeing the same line twice — so it must match."""
|
||||
call = {"id": "c1", "function": {"name": "web_fetch", "arguments": '{"url": "https://x.dev/a"}'}}
|
||||
records = _conversation()
|
||||
for _ in range(2):
|
||||
records.append({"role": "assistant", "content": "", "tool_calls": [call]})
|
||||
records.append({"role": "tool", "tool_call_id": "c1", "name": "web_fetch", "content": "Error: 403"})
|
||||
path = _write_session(tmp_path, "websocket_abc", records)
|
||||
lines = [line for line in reflect_distill.distill_session(path).text.splitlines() if "→ web_fetch" in line]
|
||||
assert len(lines) == 2
|
||||
assert lines[0] == lines[1]
|
||||
|
||||
def test_reasoning_is_shortened_not_dropped(self, tmp_path):
|
||||
records = _conversation()
|
||||
records.append({"role": "assistant", "content": "ok", "reasoning_content": "reasoning " * 200})
|
||||
path = _write_session(tmp_path, "websocket_abc", records)
|
||||
text = reflect_distill.distill_session(path).text
|
||||
assert "~ reasoning" in text
|
||||
assert len(text) < 2000
|
||||
|
||||
|
||||
class TestBatching:
|
||||
def test_batches_respect_the_character_budget(self, tmp_path):
|
||||
for i in range(6):
|
||||
_write_session(tmp_path, f"websocket_{i}", _conversation(turns=8))
|
||||
paths = reflect_distill.collect_sessions(tmp_path)
|
||||
batches = list(reflect_distill.iter_batches(paths, budget_chars=400))
|
||||
assert len(batches) > 1
|
||||
assert all(batch for batch in batches)
|
||||
|
||||
def test_oversized_session_is_not_split_or_dropped(self, tmp_path):
|
||||
"""Truncating a huge session would hide exactly the runaway loops worth finding."""
|
||||
records = _conversation()
|
||||
records.append({"role": "assistant", "content": "y" * 5000})
|
||||
_write_session(tmp_path, "websocket_big", records)
|
||||
batches = list(reflect_distill.iter_batches(reflect_distill.collect_sessions(tmp_path), budget_chars=100))
|
||||
assert len(batches) == 1
|
||||
assert len(batches[0][0]) > 5000
|
||||
|
||||
def test_collect_sessions_skips_noise(self, tmp_path):
|
||||
_write_session(tmp_path, "websocket_keep", _conversation())
|
||||
_write_session(tmp_path, "dream_drop", _conversation())
|
||||
names = [p.stem for p in reflect_distill.collect_sessions(tmp_path)]
|
||||
assert names == ["websocket_keep"]
|
||||
|
||||
def test_cursor_written_by_a_run_excludes_those_sessions_next_time(self, tmp_path):
|
||||
"""The round trip reflect_auto actually makes: digest.started becomes the next `since`.
|
||||
|
||||
A display-formatted cursor (`2026-07-11 14:02`) compares wrong against the raw ISO in
|
||||
the log, because `T` sorts above a space — every session of that day came back.
|
||||
"""
|
||||
_write_session(tmp_path, "websocket_done", _conversation())
|
||||
digests = [
|
||||
d
|
||||
for batch in reflect_distill.iter_batches(reflect_distill.collect_sessions(tmp_path), 10**6)
|
||||
for d in batch
|
||||
]
|
||||
cursor = max(digest.started for digest in digests)
|
||||
assert reflect_distill.collect_sessions(tmp_path, since=cursor) == []
|
||||
|
||||
def test_since_excludes_already_processed_sessions(self, tmp_path):
|
||||
_write_session(tmp_path, "websocket_old", _conversation())
|
||||
new = _conversation()
|
||||
new[0]["created_at"] = "2026-08-20T09:00:00"
|
||||
_write_session(tmp_path, "websocket_new", new)
|
||||
names = [p.stem for p in reflect_distill.collect_sessions(tmp_path, since="2026-08-01T00:00:00")]
|
||||
assert names == ["websocket_new"]
|
||||
Reference in New Issue
Block a user