lachtan/nanobot-runtime
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nanobot: 2026-09-10 12:33:37

Browse Source
This commit is contained in:
lachtan
2026-09-10 12:33:37 +02:00
parent a65d082b27
commit 52161b1cd3
95 changed files with 4904 additions and 6041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
skills/wiki/scripts/wiki_chunker.py Normal file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = ["markdown-it-py", "pyyaml"]
# ///
"""Structure-aware markdown chunker for the wiki skill.
Splits a document along its H1-H3 heading hierarchy, then merges small siblings and
splits oversized sections along block boundaries. Every chunk carries a breadcrumb
(`path > title > section > subsection`) which goes into the embedded text as well as
the metadata, so a vector represents a passage in context rather than in isolation.
markdown-it-py supplies the AST (it knows the CommonMark edge cases); the chunking
policy below is ours. Block boundaries come from token line maps, so the emitted text
is the original markdown — code fences and tables stay byte-for-byte intact.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
import yaml
from markdown_it import MarkdownIt # ty: ignore[unresolved-import]
CHUNKER_VERSION = "1"
MERGE_BELOW = 200
SPLIT_ABOVE = 800
OVERLAP = 64
CHARS_PER_TOKEN = 4
SECTION_LEVELS = (1, 2, 3)
BREADCRUMB_SEP = " > "
_FRONTMATTER_RE = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\Z)", re.DOTALL)
_md = MarkdownIt("commonmark")
@dataclass(frozen=True)
class Chunk:
breadcrumb: str
text: str
@dataclass(frozen=True)
class ParsedFile:
title: str | None = None
tags: list[str] = field(default_factory=list)
headings: list[str] = field(default_factory=list)
chunks: list[Chunk] = field(default_factory=list)
@dataclass(frozen=True)
class _Heading:
start: int
end: int
level: int
title: str
@dataclass(frozen=True)
class _Section:
breadcrumb: tuple[str, ...]
text: str
@property
def parent(self) -> tuple[str, ...]:
return self.breadcrumb[:-1]
def token_estimate(text: str) -> int:
"""Approximate token count. Sizing does not need a real tokenizer (plan: 4 chars/token)."""
return max(1, len(text) // CHARS_PER_TOKEN)
def parse_markdown(text: str, path: str) -> ParsedFile:
"""Chunk one markdown document. `path` is the source-relative path, the breadcrumb root."""
frontmatter, body = _split_frontmatter(text)
frontmatter_title = _frontmatter_title(frontmatter)
tags = _frontmatter_tags(frontmatter)
root: tuple[str, ...] = (path,) if frontmatter_title is None else (path, frontmatter_title)
sections, headings = _split_sections(body, root)
merged = _merge_small_siblings(sections)
chunks = [chunk for section in merged for chunk in _split_oversized(section)]
# The catalog title falls back to the first heading, because notes carry their title as
# `# H1` far more often than as frontmatter. It deliberately does NOT feed the breadcrumb
# root: the H1 already reaches the breadcrumb through the heading stack, and changing the
# root would rewrite every chunk's text and force a full reindex.
title = frontmatter_title or (headings[0] if headings else None)
return ParsedFile(title=title, tags=tags, headings=headings, chunks=chunks)
# ---------------------------------------------------------------------------
# frontmatter
# ---------------------------------------------------------------------------
def _split_frontmatter(text: str) -> tuple[dict, str]:
"""Peel off a leading YAML frontmatter block. Malformed frontmatter stays body text."""
match = _FRONTMATTER_RE.match(text)
if not match:
return {}, text
try:
data = yaml.safe_load(match.group(1))
except yaml.YAMLError:
return {}, text
if not isinstance(data, dict):
return {}, text
return data, text[match.end() :]
def _frontmatter_title(frontmatter: dict) -> str | None:
title = frontmatter.get("title")
if isinstance(title, str) and title.strip():
return title.strip()
return None
def _frontmatter_tags(frontmatter: dict) -> list[str]:
raw = frontmatter.get("tags")
if isinstance(raw, str):
values = raw.split(",")
elif isinstance(raw, list):
values = [str(item) for item in raw]
else:
return []
return [tag.strip() for tag in values if tag.strip()]
# ---------------------------------------------------------------------------
# sectioning
# ---------------------------------------------------------------------------
def _split_sections(body: str, root: tuple[str, ...]) -> tuple[list[_Section], list[str]]:
"""Cut the body at H1-H3 headings. H4+ stay inside their parent section."""
lines = body.split("\n")
tokens = _md.parse(body)
starts: list[_Heading] = []
for index, token in enumerate(tokens):
if token.type != "heading_open" or token.map is None:
continue
level = int(token.tag[1:])
if level not in SECTION_LEVELS:
continue
inline = tokens[index + 1] if index + 1 < len(tokens) else None
title = inline.content.strip() if inline is not None else ""
starts.append(_Heading(start=token.map[0], end=token.map[1], level=level, title=title))
headings = [heading.title for heading in starts]
boundaries = [heading.start for heading in starts] + [len(lines)]
sections: list[_Section] = []
preamble = "\n".join(lines[: boundaries[0]]).strip()
if preamble:
sections.append(_Section(root, preamble))
stack: list[tuple[int, str]] = []
for position, heading in enumerate(starts):
while stack and stack[-1][0] >= heading.level:
stack.pop()
stack.append((heading.level, heading.title))
section_end = boundaries[position + 1]
# A heading with no prose of its own would embed as a bare title; the heading
# still reaches the index through its children's breadcrumbs, so drop it.
if not "\n".join(lines[heading.end : section_end]).strip():
continue
text = "\n".join(lines[heading.start : section_end]).strip()
sections.append(_Section(_dedupe(root + tuple(title for _, title in stack)), text))
return sections, headings
def _dedupe(parts: tuple[str, ...]) -> tuple[str, ...]:
"""Drop consecutive repeats so a frontmatter title matching the H1 shows up once."""
out: list[str] = []
for part in parts:
if not out or out[-1] != part:
out.append(part)
return tuple(out)
def _merge_small_siblings(sections: list[_Section]) -> list[_Section]:
"""Glue a too-small section onto the following sibling under the same parent."""
merged: list[_Section] = []
for section in sections:
if not merged:
merged.append(section)
continue
previous = merged[-1]
combined = f"{previous.text}\n\n{section.text}"
if (
token_estimate(previous.text) < MERGE_BELOW
and previous.parent == section.parent
and token_estimate(combined) <= SPLIT_ABOVE
):
merged[-1] = _Section(previous.breadcrumb, combined)
else:
merged.append(section)
return merged
# ---------------------------------------------------------------------------
# splitting
# ---------------------------------------------------------------------------
def _split_oversized(section: _Section) -> list[Chunk]:
"""Break a section over SPLIT_ABOVE into block-aligned pieces with OVERLAP carry-over."""
breadcrumb = BREADCRUMB_SEP.join(section.breadcrumb)
if token_estimate(section.text) <= SPLIT_ABOVE:
return [Chunk(breadcrumb, _embed_text(breadcrumb, section.text))]
blocks = _top_level_blocks(section.text)
chunks: list[Chunk] = []
current: list[str] = []
for block in blocks:
candidate = [*current, block]
if current and token_estimate("\n\n".join(candidate)) > SPLIT_ABOVE:
chunks.append(Chunk(breadcrumb, _embed_text(breadcrumb, "\n\n".join(current))))
current = [*_overlap_tail(current), block]
else:
current = candidate
if current:
chunks.append(Chunk(breadcrumb, _embed_text(breadcrumb, "\n\n".join(current))))
return chunks
def _top_level_blocks(text: str) -> list[str]:
"""Top-level markdown blocks, taken from token line maps so fences stay whole."""
lines = text.split("\n")
tokens = _md.parse(text)
starts = sorted({token.map[0] for token in tokens if token.level == 0 and token.map})
if not starts:
return [text]
bounds = [*starts, len(lines)]
blocks = ["\n".join(lines[bounds[i] : bounds[i + 1]]).strip() for i in range(len(starts))]
return [block for block in blocks if block]
def _overlap_tail(blocks: list[str]) -> list[str]:
"""Trailing whole blocks of the emitted chunk, up to OVERLAP tokens."""
tail: list[str] = []
budget = OVERLAP
for block in reversed(blocks):
cost = token_estimate(block)
if cost > budget:
break
tail.insert(0, block)
budget -= cost
return tail
def _embed_text(breadcrumb: str, text: str) -> str:
"""The stored chunk text carries its breadcrumb, so the vector sees the context."""
return f"{breadcrumb}\n\n{text}"

225
skills/wiki/scripts/wiki_config.py Normal file
View File

@@ -0,0 +1,225 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = ["pyyaml"]
# ///
"""Layout and configuration for the wiki skill.
All runtime data lives under `workspace/wiki/`; only `config.yaml` is versioned.
The source id is the stable key — the catalog and the vectors hang off it, while a
URL or a path may change. Renaming an id is therefore an explicit invalidation of
that source's index, not a rename.
Scope precedence is `paths` (whitelist — what is not in it does not exist for the
index) then `include` (extension whitelist) then `exclude` (scalpel, wins over both).
"""
from __future__ import annotations
import os
import re
from dataclasses import dataclass
from pathlib import Path
import yaml
# workspace/skills/wiki/scripts/wiki_config.py -> parents[3] = workspace root.
WORKSPACE = Path(__file__).resolve().parents[3]
WIKI_DIR = WORKSPACE / "wiki"
DEFAULT_DB_PATH = WIKI_DIR / "index.sqlite"
DEFAULT_CONFIG_PATH = WIKI_DIR / "config.yaml"
REMOTE_DIR = WIKI_DIR / "remote"
LOCK_PATH = WIKI_DIR / ".sync.lock"
SYNC_LOG_PATH = WORKSPACE / "log" / "wiki_sync.log"
GIT_KIND = "git"
WORKSPACE_KIND = "workspace"
VALID_KINDS = (GIT_KIND, WORKSPACE_KIND)
def db_path() -> Path:
"""Index location, overridable with WIKI_DB for tests."""
return Path(os.environ.get("WIKI_DB", str(DEFAULT_DB_PATH)))
def config_path() -> Path:
"""Config location, overridable with WIKI_CONFIG for tests."""
return Path(os.environ.get("WIKI_CONFIG", str(DEFAULT_CONFIG_PATH)))
class ConfigError(ValueError):
"""Malformed or incomplete wiki/config.yaml."""
@dataclass(frozen=True)
class EmbeddingConfig:
endpoint: str
model: str
dims: int
batch: int
keep_alive: int
query_prefix: str
@dataclass(frozen=True)
class SourceConfig:
source_id: str
kind: str
url: str | None
paths: tuple[str, ...]
include: tuple[str, ...]
exclude: tuple[str, ...]
def covers(self, rel_path: str) -> bool:
"""True when a source-relative path belongs in the index."""
if not any(_matches(pattern, rel_path) for pattern in self.paths):
return False
name = rel_path.rsplit("/", 1)[-1]
if not any(_matches(pattern, name) for pattern in self.include):
return False
return not any(_matches(pattern, rel_path) for pattern in self.exclude)
def covers_dir(self, rel_dir: str) -> bool:
"""Cheap walk prune: could anything under this directory ever be covered?"""
if not rel_dir:
return True
probe = f"{rel_dir}/"
if any(_matches(pattern, probe) for pattern in self.exclude):
return False
return any(_prefix_could_match(pattern, probe) for pattern in self.paths)
@dataclass(frozen=True)
class WikiConfig:
embedding: EmbeddingConfig
sources: tuple[SourceConfig, ...]
def source(self, source_id: str) -> SourceConfig | None:
return next((s for s in self.sources if s.source_id == source_id), None)
def source_root(source: SourceConfig) -> Path:
"""Where a source's files live. Git clones are derived from the id, never configured."""
if source.kind == GIT_KIND:
return REMOTE_DIR / source.source_id
return WORKSPACE
def load_config(path: Path | None = None) -> WikiConfig:
path = path or config_path()
if not path.exists():
raise ConfigError(f"missing config: {path}")
try:
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
except yaml.YAMLError as exc:
raise ConfigError(f"unparseable config {path}: {exc}") from exc
if not isinstance(raw, dict):
raise ConfigError(f"config {path} must be a mapping")
return WikiConfig(
embedding=_parse_embedding(raw.get("embedding")),
sources=_parse_sources(raw.get("sources")),
)
def _as_mapping(raw: object, what: str) -> dict[str, object]:
if not isinstance(raw, dict):
raise ConfigError(f"{what} must be a mapping")
return {str(key): value for key, value in raw.items()}
def _parse_embedding(raw: object) -> EmbeddingConfig:
values = _as_mapping(raw, "`embedding`")
missing = [key for key in ("endpoint", "model", "dims") if key not in values]
if missing:
raise ConfigError(f"embedding is missing {', '.join(missing)}")
keep_alive = values.get("keep_alive", -1)
if not isinstance(keep_alive, int):
# Ollama rejects a string keep_alive of "-1" with HTTP 400.
raise ConfigError("embedding.keep_alive must be a number, not a string")
return EmbeddingConfig(
endpoint=str(values["endpoint"]).rstrip("/"),
model=str(values["model"]),
dims=int(str(values["dims"])),
batch=int(str(values.get("batch", 32))),
keep_alive=keep_alive,
query_prefix=str(values.get("query_prefix", "")),
)
def _parse_sources(raw: object) -> tuple[SourceConfig, ...]:
entries = _as_mapping(raw, "`sources`")
if not entries:
raise ConfigError("config needs a non-empty `sources` mapping")
sources = []
for source_id, raw_body in entries.items():
body = _as_mapping(raw_body, f"source {source_id}")
kind = str(body.get("kind"))
if kind not in VALID_KINDS:
raise ConfigError(f"source {source_id}: kind must be one of {VALID_KINDS}, got {kind!r}")
url = body.get("url")
if kind == GIT_KIND and not url:
raise ConfigError(f"source {source_id}: git sources need a url")
sources.append(
SourceConfig(
source_id=source_id,
kind=kind,
url=str(url) if url else None,
paths=_as_patterns(body.get("paths"), source_id, "paths"),
include=_as_patterns(body.get("include") or ["*.md"], source_id, "include"),
exclude=_as_patterns(body.get("exclude") or [], source_id, "exclude"),
)
)
return tuple(sources)
def _as_patterns(raw: object, source_id: str, key: str) -> tuple[str, ...]:
if raw is None:
raise ConfigError(f"source {source_id}: `{key}` is required")
if not isinstance(raw, list):
raise ConfigError(f"source {source_id}: `{key}` must be a list")
return tuple(str(item) for item in raw)
# ---------------------------------------------------------------------------
# glob matching
#
# fnmatch lets `*` cross a `/` and PurePath.match has no recursive `**` before
# Python 3.13, so the patterns are translated by hand.
# ---------------------------------------------------------------------------
_SEGMENT_ANY = "[^/]*"
def _glob_to_regex(pattern: str) -> str:
parts = pattern.split("/")
out = []
for index, part in enumerate(parts):
is_last = index == len(parts) - 1
if part == "**":
out.append(".*" if is_last else "(?:[^/]+/)*")
else:
segment = re.escape(part).replace(r"\*", _SEGMENT_ANY).replace(r"\?", "[^/]")
out.append(segment if is_last else segment + "/")
return "^" + "".join(out) + "$"
def _matches(pattern: str, value: str) -> bool:
return re.match(_glob_to_regex(pattern), value) is not None
def _prefix_could_match(pattern: str, directory: str) -> bool:
"""True when `pattern` can still match something below `directory`."""
if pattern.startswith("**"):
return True
pattern_parts = pattern.split("/")
dir_parts = [part for part in directory.split("/") if part]
for depth, dir_part in enumerate(dir_parts):
if depth >= len(pattern_parts):
return False
pattern_part = pattern_parts[depth]
if pattern_part == "**":
return True
if not _matches(pattern_part, dir_part):
return False
return True

129
skills/wiki/scripts/wiki_db.py Normal file
View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = ["sqlite-vec==0.1.6"]
# ///
"""SQLite storage layer for the wiki skill.
Schema, connection factory and the sqlite-vec extension load.
`chunks` is the canonical retrieval unit: `chunks_fts` gives it a BM25 rank and
`vec_chunks` a KNN rank, both keyed on `chunks.id`, so RRF merges two rankings of
the *same* set.
"""
from __future__ import annotations
import sqlite3
from pathlib import Path
import sqlite_vec # ty: ignore[unresolved-import]
# Compiled into the vec0 table definition. `wiki_embed` guards config.dims against it —
# a mismatch means the index was built for a different model.
EMBEDDING_DIMS = 1024
SCHEMA = f"""
PRAGMA journal_mode = WAL;
PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS sources (
source_id TEXT PRIMARY KEY,
kind TEXT NOT NULL CHECK(kind IN ('git', 'workspace')),
indexed_rev TEXT,
last_sync_at TEXT
);
CREATE TABLE IF NOT EXISTS files (
source_id TEXT NOT NULL REFERENCES sources(source_id),
path TEXT NOT NULL,
title TEXT,
tags TEXT,
headings TEXT,
sha256 TEXT NOT NULL,
size INTEGER NOT NULL,
mtime REAL,
indexed_at TEXT NOT NULL,
PRIMARY KEY (source_id, path)
);
CREATE TABLE IF NOT EXISTS chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_id TEXT NOT NULL,
path TEXT NOT NULL,
chunk_idx INTEGER NOT NULL,
breadcrumb TEXT NOT NULL,
text TEXT NOT NULL,
embedded_at TEXT,
UNIQUE (source_id, path, chunk_idx),
FOREIGN KEY (source_id, path) REFERENCES files(source_id, path) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_chunks_pending ON chunks(id) WHERE embedded_at IS NULL;
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
breadcrumb, text,
content='chunks', content_rowid='id',
tokenize='unicode61 remove_diacritics 2'
);
CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, breadcrumb, text) VALUES (new.id, new.breadcrumb, new.text);
END;
CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, breadcrumb, text)
VALUES('delete', old.id, old.breadcrumb, old.text);
END;
-- The WHEN guard keeps `UPDATE chunks SET embedded_at` (sync step 4b) from rewriting
-- an FTS row whose indexed text did not change.
CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks
WHEN old.text IS NOT new.text OR old.breadcrumb IS NOT new.breadcrumb
BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, breadcrumb, text)
VALUES('delete', old.id, old.breadcrumb, old.text);
INSERT INTO chunks_fts(rowid, breadcrumb, text) VALUES (new.id, new.breadcrumb, new.text);
END;
CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
embedding float[{EMBEDDING_DIMS}] distance_metric=cosine
);
"""
def get_db(path: Path) -> sqlite3.Connection:
"""Return an autocommit connection with sqlite-vec loaded and foreign keys on."""
conn = sqlite3.connect(path, isolation_level=None)
conn.enable_load_extension(True)
sqlite_vec.load(conn)
conn.enable_load_extension(False)
conn.execute("PRAGMA journal_mode = WAL")
conn.execute("PRAGMA foreign_keys = ON")
conn.row_factory = sqlite3.Row
_migrate(conn)
return conn
def _migrate(conn: sqlite3.Connection) -> None:
"""Idempotently bring an existing DB up to the current schema.
init_db only runs the full SCHEMA on a missing file, so live DBs never see
later additions. Each step must be a no-op once applied.
"""
# No migrations yet — schema_version 1 is the initial shape.
def init_db(path: Path) -> None:
"""Create tables, indexes and triggers if they don't exist."""
path.parent.mkdir(parents=True, exist_ok=True)
conn = get_db(path)
try:
conn.executescript(SCHEMA)
finally:
conn.close()

127
skills/wiki/scripts/wiki_embed.py Normal file
View File

@@ -0,0 +1,127 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = ["requests", "pyyaml", "markdown-it-py", "sqlite-vec==0.1.6"]
# ///
"""Embedding client and the identity guard over the vector space.
Two invariants live here:
* Vectors are stored L2-normalized, so cosine distance equals the dot product.
* The query prefix must be bit-identical at index and at query time. That is the real
reason it is persisted in `meta` rather than only read from the config — mixing
vectors produced under two different contracts degrades results silently, which is
the most expensive kind of bug.
An unreachable Ollama is not an error here: the caller stores chunks with
`embedded_at IS NULL` and the query side says out loud that it is FTS-only.
"""
from __future__ import annotations
import math
from collections.abc import Sequence
from dataclasses import dataclass
import requests
from wiki_chunker import CHUNKER_VERSION
from wiki_config import EmbeddingConfig
from wiki_db import EMBEDDING_DIMS
SCHEMA_VERSION = "1"
SQLITE_VEC_VERSION = "0.1.6"
REQUEST_TIMEOUT_SECONDS = 60
# Only these keys make two vectors comparable; the rest of `meta` is informational.
GUARDED_META_KEYS = (
"embedding_model",
"embedding_dims",
"normalized",
"query_prefix",
"chunker_version",
)
class EmbeddingUnavailable(RuntimeError):
"""Ollama could not be reached or refused the request."""
class IndexIdentityMismatch(RuntimeError):
"""The index was built under a different embedding contract — a reindex is needed."""
def expected_meta(config: EmbeddingConfig) -> dict[str, str]:
return {
"embedding_model": config.model,
"embedding_dims": str(config.dims),
"normalized": "l2",
"query_prefix": config.query_prefix,
"chunker_version": CHUNKER_VERSION,
"schema_version": SCHEMA_VERSION,
"sqlite_vec_version": SQLITE_VEC_VERSION,
}
def meta_mismatches(stored: dict[str, str], config: EmbeddingConfig) -> list[str]:
"""Guarded meta keys that disagree with the config. Empty on a fresh (unwritten) index."""
if not stored:
return []
expected = expected_meta(config)
return [key for key in GUARDED_META_KEYS if stored.get(key) != expected[key]]
def require_matching_index(stored: dict[str, str], config: EmbeddingConfig) -> None:
"""Refuse to query an index built under a different contract."""
if config.dims != EMBEDDING_DIMS:
raise IndexIdentityMismatch(f"reindex needed: config dims {config.dims} != schema dims {EMBEDDING_DIMS}")
mismatches = meta_mismatches(stored, config)
if mismatches:
detail = ", ".join(
f"{key}: index={stored.get(key)!r} config={expected_meta(config)[key]!r}" for key in mismatches
)
raise IndexIdentityMismatch(f"reindex needed: {detail}")
def l2_normalize(vector: Sequence[float]) -> list[float]:
norm = math.sqrt(sum(value * value for value in vector))
if norm == 0.0:
return list(vector)
return [value / norm for value in vector]
@dataclass(frozen=True)
class OllamaEmbedder:
config: EmbeddingConfig
def embed_documents(self, texts: Sequence[str]) -> list[list[float]]:
"""Documents are embedded without the instruct prefix (the model is asymmetric)."""
return self._embed(list(texts))
def embed_query(self, text: str) -> list[float]:
return self._embed([self.config.query_prefix + text])[0]
def probe(self) -> None:
"""Raise EmbeddingUnavailable unless the endpoint answers an embed request."""
self._embed(["ping"])
def _embed(self, inputs: list[str]) -> list[list[float]]:
if not inputs:
return []
payload = {
"model": self.config.model,
"input": inputs,
"keep_alive": self.config.keep_alive,
}
try:
response = requests.post(f"{self.config.endpoint}/api/embed", json=payload, timeout=REQUEST_TIMEOUT_SECONDS)
response.raise_for_status()
embeddings = response.json()["embeddings"]
except (requests.RequestException, KeyError, ValueError) as exc:
raise EmbeddingUnavailable(f"{self.config.endpoint}: {exc}") from exc
if len(embeddings) != len(inputs):
raise EmbeddingUnavailable(f"asked for {len(inputs)} vectors, got {len(embeddings)}")
for vector in embeddings:
if len(vector) != self.config.dims:
raise EmbeddingUnavailable(f"model returned {len(vector)} dims, config says {self.config.dims}")
return [l2_normalize(vector) for vector in embeddings]

308
skills/wiki/scripts/wiki_search.py Normal file
View File

@@ -0,0 +1,308 @@
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.11"
# dependencies = ["requests", "pyyaml", "markdown-it-py", "sqlite-vec==0.1.6"]
# ///
"""Query side of the wiki skill — three layers, cheapest first.
grep live ripgrep over the files on disk. No index, never stale, and not limited
to the indexed extensions, so it is the layer that covers source code.
toc directory -> file -> title + tags, read straight from the `files` catalog.
search FTS5 (BM25) and vec0 (KNN) over the same `chunks` rows, merged with RRF.
Both halves rank the same unit, which is what makes the merge meaningful. The
parameters below are module constants on purpose: `--limit` is the only knob worth
exposing, and a config key that never changes is a config key that rots.
"""
from __future__ import annotations
import argparse
import json
import shutil
import sqlite3
import subprocess
import sys
from pathlib import Path
import wiki_store as store
from wiki_config import (
GIT_KIND,
ConfigError,
SourceConfig,
WikiConfig,
db_path,
load_config,
source_root,
)
from wiki_embed import (
EmbeddingUnavailable,
IndexIdentityMismatch,
OllamaEmbedder,
require_matching_index,
)
CANDIDATE_LIMIT = 50 # KNN k, and the LIMIT for BM25
RRF_K = 60 # Cormack et al. 2009
DEFAULT_LIMIT = 10
PREFIX_MIN_LENGTH = 3 # a one- or two-character prefix matches too widely to carry signal
EXCERPT_CHARS = 320
GREP_TIMEOUT = 30
GREP_MAX_PER_FILE = 5
def rrf_merge(ranked_lists: list[list[int]]) -> list[tuple[int, float]]:
"""score(id) = sum over lists of 1 / (RRF_K + rank). No weights: both halves count equally."""
scores: dict[int, float] = {}
for ids in ranked_lists:
for rank, chunk_id in enumerate(ids, start=1):
scores[chunk_id] = scores.get(chunk_id, 0.0) + 1.0 / (RRF_K + rank)
return sorted(scores.items(), key=lambda item: (-item[1], item[0]))
def fts_match_expression(query: str) -> str:
"""Turn free text into an FTS5 OR-query of quoted prefix terms.
Terms are quoted so reserved words (`and`, `not`, `near`) and punctuation cannot be
read as operators. The trailing `*` covers Czech inflection, which `unicode61` does
not stem — `záloh*` finds záloha/zálohování/zálohy.
"""
terms = []
for word in _tokenize(query):
wildcard = "*" if len(word) >= PREFIX_MIN_LENGTH else ""
terms.append(f'"{word}"{wildcard}')
return " OR ".join(terms)
def _tokenize(text: str) -> list[str]:
return "".join(char if char.isalnum() else " " for char in text).split()
# ---------------------------------------------------------------------------
# search
# ---------------------------------------------------------------------------
def run_search(config: WikiConfig, query: str, limit: int) -> int:
expression = fts_match_expression(query)
if not expression:
print("(empty query)")
return 0
with store.connection(db_path()) as conn:
try:
require_matching_index(store.read_meta(conn), config.embedding)
except IndexIdentityMismatch as exc:
print(str(exc), file=sys.stderr)
return 1
bm25_ids = store.bm25_ranked_ids(conn, expression, CANDIDATE_LIMIT)
vector_ids: list[int] = []
degraded: str | None = None
try:
vector = OllamaEmbedder(config.embedding).embed_query(query)
vector_ids = store.knn_ranked_ids(conn, vector, CANDIDATE_LIMIT)
except EmbeddingUnavailable as exc:
degraded = f"note: embeddings unavailable ({exc}) — FTS-only results"
pending = store.index_stats(conn)["pending"]
merged = rrf_merge([ids for ids in (bm25_ids, vector_ids) if ids])[:limit]
rows = store.fetch_chunks(conn, [chunk_id for chunk_id, _ in merged])
if degraded:
print(degraded)
elif pending:
print(f"note: {pending} chunks still awaiting vectors — semantic half is incomplete")
if not merged:
print("(no matches)")
return 0
bm25_rank = {chunk_id: rank for rank, chunk_id in enumerate(bm25_ids, start=1)}
vector_rank = {chunk_id: rank for rank, chunk_id in enumerate(vector_ids, start=1)}
for position, (chunk_id, score) in enumerate(merged, start=1):
row = rows.get(chunk_id)
if row is None:
continue
origin = _origin_label(bm25_rank.get(chunk_id), vector_rank.get(chunk_id))
print(f"{position}. {row['source_id']}:{row['path']} (rrf {score:.4f}, {origin})")
print(f" {row['breadcrumb']}")
print(_indent(_excerpt(row["text"], row["breadcrumb"])))
print()
return 0
def _origin_label(bm25: int | None, vector: int | None) -> str:
parts = []
if bm25 is not None:
parts.append(f"bm25 #{bm25}")
if vector is not None:
parts.append(f"vec #{vector}")
return ", ".join(parts)
def _excerpt(text: str, breadcrumb: str) -> str:
body = text[len(breadcrumb) :].lstrip("\n") if text.startswith(breadcrumb) else text
body = " ".join(body.split())
return body[:EXCERPT_CHARS] + ("…" if len(body) > EXCERPT_CHARS else "")
def _indent(text: str) -> str:
return "\n".join(f" {line}" for line in text.splitlines())
# ---------------------------------------------------------------------------
# toc
# ---------------------------------------------------------------------------
def run_toc(source_id: str | None, tag: str | None) -> int:
with store.connection(db_path()) as conn:
rows = store.list_toc_files(conn, source_id=source_id, tag=tag)
if not rows:
print("(no indexed files match)")
return 0
by_source: dict[str, list[sqlite3.Row]] = {}
for row in rows:
by_source.setdefault(row["source_id"], []).append(row)
for source, files in by_source.items():
print(f"{source} ({len(files)} files)")
print()
directory = None
for row in files:
path = row["path"]
parent, _, name = path.rpartition("/")
if parent != directory:
directory = parent
print(f"{parent}/" if parent else "./")
print(f" {name:<28} {row['title'] or '':<32} {_tag_label(row['tags'])}".rstrip())
print()
return 0
def _tag_label(raw: str | None) -> str:
try:
tags = json.loads(raw) if raw else []
except json.JSONDecodeError:
return ""
return f"[{', '.join(tags)}]" if tags else ""
# ---------------------------------------------------------------------------
# grep
# ---------------------------------------------------------------------------
def grep_roots(source: SourceConfig) -> list[Path]:
"""Directories ripgrep should walk for one source.
A git clone is searched whole. A workspace source is bounded by the literal prefix
of each `paths` glob, so grep stays inside what the source claims without being
narrowed to the indexed extensions.
"""
root = source_root(source)
if source.kind == GIT_KIND:
return [root] if root.is_dir() else []
roots = []
for pattern in source.paths:
prefix = _literal_prefix(pattern)
candidate = root / prefix if prefix else root
if candidate.is_dir() and candidate not in roots:
roots.append(candidate)
return roots
def _literal_prefix(pattern: str) -> str:
parts = []
for segment in pattern.split("/"):
if any(char in segment for char in "*?["):
break
parts.append(segment)
return "/".join(parts)
def run_grep(config: WikiConfig, pattern: str, source_id: str | None) -> int:
if shutil.which("rg") is None:
print("ripgrep (rg) not found", file=sys.stderr)
return 1
sources = [s for s in config.sources if not source_id or s.source_id == source_id]
if not sources:
print(f"unknown source {source_id!r}", file=sys.stderr)
return 1
roots = [path for source in sources for path in grep_roots(source)]
if not roots:
print("(nothing on disk to grep — has wiki_sync.py run?)")
return 0
command = [
"rg",
"--line-number",
"--no-heading",
"--color",
"never",
"--smart-case",
"--max-count",
str(GREP_MAX_PER_FILE),
"--glob",
"!.git",
pattern,
*[str(path) for path in roots],
]
try:
result = subprocess.run(command, capture_output=True, text=True, timeout=GREP_TIMEOUT, check=False)
except subprocess.TimeoutExpired:
print(f"grep timed out after {GREP_TIMEOUT}s", file=sys.stderr)
return 1
if result.returncode not in (0, 1):
print(result.stderr.strip(), file=sys.stderr)
return 1
output = result.stdout.strip()
print(output if output else "(no matches)")
return 0
# ---------------------------------------------------------------------------
# cli
# ---------------------------------------------------------------------------
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Search the wiki index over your notes.")
subparsers = parser.add_subparsers(dest="command", required=True)
search = subparsers.add_parser("search", help="hybrid BM25 + vector search over chunks")
search.add_argument("query", help="free text; Czech inflection is covered by prefix matching")
search.add_argument("--limit", type=int, default=DEFAULT_LIMIT, help="chunks to return")
toc = subparsers.add_parser("toc", help="directory -> file -> title + tags from the catalog")
toc.add_argument("--source", help="limit to one source id")
toc.add_argument("--tag", help="only files carrying this frontmatter tag")
grep = subparsers.add_parser("grep", help="live ripgrep over the files on disk, index-free")
grep.add_argument("pattern", help="ripgrep regex")
grep.add_argument("--source", help="limit to one source id")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
try:
config = load_config()
except ConfigError as exc:
print(f"config error: {exc}", file=sys.stderr)
return 1
if args.command == "search":
return run_search(config, args.query, args.limit)
if args.command == "toc":
return run_toc(args.source, args.tag)
return run_grep(config, args.pattern, args.source)
if __name__ == "__main__":
sys.exit(main())

280
skills/wiki/scripts/wiki_store.py Normal file
View File

@@ -0,0 +1,280 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.11"
# dependencies = ["sqlite-vec==0.1.6"]
# ///
"""Data-access layer for the wiki skill.
Pure SQL plus lifecycle helpers. No printing, no argparse, no sys.exit.
`vec_chunks` is a vec0 virtual table and therefore NOT reachable by the foreign key
cascade that cleans up `chunks` and `chunks_fts`. Every path that removes chunks must
delete their vectors explicitly — that is why the deletes here go through
`_delete_vectors_for_file`.
"""
from __future__ import annotations
import json
import sqlite3
from collections.abc import Iterator, Sequence
from contextlib import contextmanager
from pathlib import Path
from sqlite_vec import serialize_float32 # ty: ignore[unresolved-import]
from wiki_db import get_db, init_db
@contextmanager
def connection(db_path: Path) -> Iterator[sqlite3.Connection]:
"""Open a connection, initialising the DB if missing."""
if not db_path.exists():
init_db(db_path)
conn = get_db(db_path)
try:
yield conn
finally:
conn.close()
@contextmanager
def tx(conn: sqlite3.Connection) -> Iterator[sqlite3.Connection]:
"""Wrap an already-open connection in an explicit transaction.
The connection is in autocommit mode (`isolation_level=None`), so transactions are
ours to open — sqlite3's implicit handling would otherwise fail a nested BEGIN.
"""
conn.execute("BEGIN")
try:
yield conn
conn.execute("COMMIT")
except Exception:
conn.execute("ROLLBACK")
raise
@contextmanager
def transaction(db_path: Path) -> Iterator[sqlite3.Connection]:
"""Open a connection wrapped in an explicit transaction."""
with connection(db_path) as conn, tx(conn):
yield conn
# ---------------------------------------------------------------------------
# meta — identity of the embedding space
# ---------------------------------------------------------------------------
def read_meta(conn: sqlite3.Connection) -> dict[str, str]:
return {row["key"]: row["value"] for row in conn.execute("SELECT key, value FROM meta")}
def write_meta(conn: sqlite3.Connection, values: dict[str, str]) -> None:
conn.executemany(
"INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
sorted(values.items()),
)
# ---------------------------------------------------------------------------
# sources
# ---------------------------------------------------------------------------
def upsert_source(conn: sqlite3.Connection, source_id: str, kind: str) -> None:
conn.execute(
"INSERT INTO sources (source_id, kind) VALUES (?, ?) ON CONFLICT(source_id) DO UPDATE SET kind = excluded.kind",
(source_id, kind),
)
def get_source(conn: sqlite3.Connection, source_id: str) -> sqlite3.Row | None:
return conn.execute("SELECT * FROM sources WHERE source_id = ?", (source_id,)).fetchone()
def mark_synced(conn: sqlite3.Connection, source_id: str, indexed_rev: str | None, now: str) -> None:
conn.execute(
"UPDATE sources SET indexed_rev = ?, last_sync_at = ? WHERE source_id = ?",
(indexed_rev, now, source_id),
)
# ---------------------------------------------------------------------------
# files
# ---------------------------------------------------------------------------
def get_file(conn: sqlite3.Connection, source_id: str, path: str) -> sqlite3.Row | None:
return conn.execute("SELECT * FROM files WHERE source_id = ? AND path = ?", (source_id, path)).fetchone()
def list_source_files(conn: sqlite3.Connection, source_id: str) -> dict[str, sqlite3.Row]:
"""Indexed files of one source, keyed by path — the basis for deletion detection."""
rows = conn.execute("SELECT * FROM files WHERE source_id = ?", (source_id,))
return {row["path"]: row for row in rows}
def upsert_file(
conn: sqlite3.Connection,
source_id: str,
path: str,
title: str | None,
tags: list[str],
headings: list[str],
sha256: str,
size: int,
mtime: float | None,
now: str,
) -> None:
conn.execute(
"INSERT INTO files (source_id, path, title, tags, headings, sha256, size, mtime, indexed_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
"ON CONFLICT(source_id, path) DO UPDATE SET "
"title = excluded.title, tags = excluded.tags, headings = excluded.headings, "
"sha256 = excluded.sha256, size = excluded.size, mtime = excluded.mtime, "
"indexed_at = excluded.indexed_at",
(
source_id,
path,
title,
json.dumps(tags, ensure_ascii=False),
json.dumps(headings, ensure_ascii=False),
sha256,
size,
mtime,
now,
),
)
def touch_file_stat(conn: sqlite3.Connection, source_id: str, path: str, size: int, mtime: float | None) -> None:
"""Refresh the cheap change-detection stats after a sha256 match (content unchanged)."""
conn.execute(
"UPDATE files SET size = ?, mtime = ? WHERE source_id = ? AND path = ?",
(size, mtime, source_id, path),
)
def delete_file(conn: sqlite3.Connection, source_id: str, path: str) -> None:
"""Drop a file and everything derived from it, vec0 rows included."""
_delete_vectors_for_file(conn, source_id, path)
conn.execute("DELETE FROM files WHERE source_id = ? AND path = ?", (source_id, path))
def list_toc_files(conn: sqlite3.Connection, source_id: str | None = None, tag: str | None = None) -> list[sqlite3.Row]:
sql = "SELECT source_id, path, title, tags FROM files"
clauses: list[str] = []
params: list[str] = []
if source_id:
clauses.append("source_id = ?")
params.append(source_id)
if tag:
clauses.append("EXISTS (SELECT 1 FROM json_each(files.tags) WHERE value = ?)")
params.append(tag)
if clauses:
sql += " WHERE " + " AND ".join(clauses)
sql += " ORDER BY source_id, path"
return list(conn.execute(sql, params))
# ---------------------------------------------------------------------------
# chunks
# ---------------------------------------------------------------------------
def _delete_vectors_for_file(conn: sqlite3.Connection, source_id: str, path: str) -> None:
ids = [
row["id"] for row in conn.execute("SELECT id FROM chunks WHERE source_id = ? AND path = ?", (source_id, path))
]
conn.executemany("DELETE FROM vec_chunks WHERE rowid = ?", [(i,) for i in ids])
def replace_chunks(conn: sqlite3.Connection, source_id: str, path: str, chunks: Sequence[tuple[str, str]]) -> None:
"""Swap a file's chunks for a freshly built set, as (breadcrumb, text) in order.
New chunks land with `embedded_at IS NULL`; the embed pass picks them up, so an
unreachable Ollama degrades to FTS-only instead of failing the sync.
"""
_delete_vectors_for_file(conn, source_id, path)
conn.execute("DELETE FROM chunks WHERE source_id = ? AND path = ?", (source_id, path))
conn.executemany(
"INSERT INTO chunks (source_id, path, chunk_idx, breadcrumb, text) VALUES (?, ?, ?, ?, ?)",
[(source_id, path, idx, breadcrumb, text) for idx, (breadcrumb, text) in enumerate(chunks)],
)
def pending_chunks(conn: sqlite3.Connection, limit: int) -> list[sqlite3.Row]:
return list(
conn.execute(
"SELECT id, breadcrumb, text FROM chunks WHERE embedded_at IS NULL ORDER BY id LIMIT ?",
(limit,),
)
)
def has_pending(conn: sqlite3.Connection) -> bool:
return conn.execute("SELECT 1 FROM chunks WHERE embedded_at IS NULL LIMIT 1").fetchone() is not None
def store_embedding(conn: sqlite3.Connection, chunk_id: int, vector: Sequence[float], now: str) -> None:
conn.execute("DELETE FROM vec_chunks WHERE rowid = ?", (chunk_id,))
conn.execute(
"INSERT INTO vec_chunks (rowid, embedding) VALUES (?, ?)",
(chunk_id, serialize_float32(list(vector))),
)
conn.execute("UPDATE chunks SET embedded_at = ? WHERE id = ?", (now, chunk_id))
def reset_index(conn: sqlite3.Connection) -> None:
"""Drop every indexed artifact, keeping the source rows. Used by `--full`."""
ids = [row["id"] for row in conn.execute("SELECT id FROM chunks")]
conn.executemany("DELETE FROM vec_chunks WHERE rowid = ?", [(i,) for i in ids])
conn.execute("DELETE FROM files")
conn.execute("UPDATE sources SET indexed_rev = NULL")
def index_stats(conn: sqlite3.Connection) -> dict[str, int]:
return {
"files": conn.execute("SELECT count(*) AS n FROM files").fetchone()["n"],
"chunks": conn.execute("SELECT count(*) AS n FROM chunks").fetchone()["n"],
"vectors": conn.execute("SELECT count(*) AS n FROM vec_chunks").fetchone()["n"],
"pending": conn.execute("SELECT count(*) AS n FROM chunks WHERE embedded_at IS NULL").fetchone()["n"],
}
def orphan_vector_ids(conn: sqlite3.Connection) -> list[int]:
"""Vector rowids with no surviving chunk — must always be empty (regression guard)."""
rows = conn.execute("SELECT rowid AS rid FROM vec_chunks WHERE rowid NOT IN (SELECT id FROM chunks)")
return [row["rid"] for row in rows]
# ---------------------------------------------------------------------------
# retrieval
# ---------------------------------------------------------------------------
def bm25_ranked_ids(conn: sqlite3.Connection, match_expr: str, limit: int) -> list[int]:
rows = conn.execute(
"SELECT rowid AS rid FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY rank LIMIT ?",
(match_expr, limit),
)
return [row["rid"] for row in rows]
def knn_ranked_ids(conn: sqlite3.Connection, vector: Sequence[float], k: int) -> list[int]:
rows = conn.execute(
"SELECT rowid AS rid FROM vec_chunks WHERE embedding MATCH ? AND k = ? ORDER BY distance",
(serialize_float32(list(vector)), k),
)
return [row["rid"] for row in rows]
def fetch_chunks(conn: sqlite3.Connection, ids: Sequence[int]) -> dict[int, sqlite3.Row]:
if not ids:
return {}
placeholders = ",".join("?" * len(ids))
rows = conn.execute(
f"SELECT id, source_id, path, chunk_idx, breadcrumb, text FROM chunks WHERE id IN ({placeholders})",
list(ids),
)
return {row["id"]: row for row in rows}

529
skills/wiki/scripts/wiki_sync.py Normal file
View File

@@ -0,0 +1,529 @@
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.11"
# dependencies = ["requests", "pyyaml", "markdown-it-py", "sqlite-vec==0.1.6"]
# ///
"""Offline indexer for the wiki skill. Runs every minute from the nanobot crontab.
Indexing never happens inside an agent turn: the `exec` tool times out at 60 s while a
full index of 10^4 chunks takes ~93 s. The agent only ever reads a finished index.
Per tick:
1. take `wiki/.sync.lock`; a live holder means exit 0 in silence
2. detect changes cheaply — `git ls-remote` (no fetch) for git sources, a
path+size+mtime walk for workspace sources, sha256 only on a stat mismatch
3. nothing changed and nothing pending -> exit 0 (the overwhelming majority of ticks)
4. re-chunk changed files, then drain every chunk with `embedded_at IS NULL` — which
is also the way back out of degraded mode after Ollama returns
5. record indexed_rev / last_sync_at
6. log a coverage line naming top-level directories no source covers
7. append the run summary to log/wiki_sync.log
Network failure is per-source: a timeout or non-zero git exit logs WARN, skips that
source with its `indexed_rev` untouched, and lets the others finish. Without the
timeouts a hanging `ls-remote` would hold the lock and block workspace sources that
have nothing to do with the network.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import subprocess
import sys
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
import wiki_store as store
from wiki_chunker import parse_markdown
from wiki_config import (
GIT_KIND,
LOCK_PATH,
SYNC_LOG_PATH,
WIKI_DIR,
ConfigError,
SourceConfig,
WikiConfig,
db_path,
load_config,
source_root,
)
from wiki_embed import (
EmbeddingUnavailable,
OllamaEmbedder,
expected_meta,
meta_mismatches,
)
STALE_SECONDS = 30 * 60
LS_REMOTE_TIMEOUT = 20
GIT_TIMEOUT = 300
GIT_ERROR_CHARS = 300
SKIP_DIRS = frozenset({".git"})
class GitError(RuntimeError):
"""A git subprocess failed or timed out."""
@dataclass
class SourcePlan:
"""What one source needs done this tick."""
changed: list[str] = field(default_factory=list)
deleted: list[str] = field(default_factory=list)
stat_refresh: list[tuple[str, int, float]] = field(default_factory=list)
new_rev: str | None = None
@property
def has_work(self) -> bool:
return bool(self.changed or self.deleted or self.stat_refresh)
def log(message: str) -> None:
SYNC_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
stamp = datetime.now().astimezone().isoformat(timespec="seconds")
with SYNC_LOG_PATH.open("a", encoding="utf-8") as handle:
handle.write(f"{stamp} {message}\n")
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
# ---------------------------------------------------------------------------
# lock
# ---------------------------------------------------------------------------
def _pid_alive(pid: int) -> bool:
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def _lock_is_stale() -> bool:
"""A lock is dead if unreadable, its pid is gone, or it is older than STALE_SECONDS."""
try:
data = json.loads(LOCK_PATH.read_text(encoding="utf-8"))
pid = int(data["pid"])
started = datetime.fromisoformat(data["started_at"])
except (OSError, ValueError, KeyError):
return True
if not _pid_alive(pid):
return True
return (datetime.now().astimezone() - started).total_seconds() > STALE_SECONDS
def acquire_lock() -> bool:
"""Atomically create the lock. False when a live sync already runs.
Reclaiming a stale lock is not optional: without it a crashed run (OOM, reboot)
would make every later cron tick exit 0 in silence, forever.
"""
LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
for _ in range(2):
try:
handle = os.open(LOCK_PATH, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
except FileExistsError:
if not _lock_is_stale():
return False
log("WARN stale lock, reclaiming")
LOCK_PATH.unlink(missing_ok=True)
continue
payload = {"pid": os.getpid(), "started_at": datetime.now().astimezone().isoformat()}
with os.fdopen(handle, "w", encoding="utf-8") as file:
json.dump(payload, file)
return True
return False
def release_lock() -> None:
LOCK_PATH.unlink(missing_ok=True)
# ---------------------------------------------------------------------------
# filesystem
# ---------------------------------------------------------------------------
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(65536), b""):
digest.update(block)
return digest.hexdigest()
def walk_source(source: SourceConfig, root: Path) -> dict[str, tuple[int, float]]:
"""Covered files under `root`, as rel_path -> (size, mtime). Prunes uncovered dirs."""
found: dict[str, tuple[int, float]] = {}
if not root.is_dir():
return found
for dirpath, dirnames, filenames in os.walk(root):
rel_dir = os.path.relpath(dirpath, root)
rel_dir = "" if rel_dir == "." else rel_dir
dirnames[:] = [
name for name in dirnames if name not in SKIP_DIRS and source.covers_dir(f"{rel_dir}/{name}".lstrip("/"))
]
for name in filenames:
rel_path = f"{rel_dir}/{name}".lstrip("/")
if not source.covers(rel_path):
continue
file_path = Path(dirpath) / name
if not file_path.is_file():
continue
stat = file_path.stat()
found[rel_path] = (stat.st_size, stat.st_mtime)
return found
def uncovered_top_level_dirs(source: SourceConfig, root: Path) -> list[str]:
"""Top-level directories holding markdown that no source path reaches.
This is the safety net under the whitelist: a directory that silently fails to be
indexed shows up here instead of nowhere. `wiki/` is skipped because it holds this
skill's own clones and index — never a candidate, so reporting it is pure noise.
"""
if not root.is_dir():
return []
uncovered = []
for entry in sorted(root.iterdir()):
if not entry.is_dir() or entry.name in SKIP_DIRS or entry == WIKI_DIR:
continue
if source.covers_dir(entry.name):
continue
if next(entry.rglob("*.md"), None) is not None:
uncovered.append(entry.name)
return uncovered
# ---------------------------------------------------------------------------
# git driver
# ---------------------------------------------------------------------------
def _git(args: list[str], cwd: Path | None = None, timeout: int = GIT_TIMEOUT) -> str:
try:
result = subprocess.run(
["git", *args],
cwd=cwd,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired as exc:
raise GitError(f"git {' '.join(args)} timed out after {timeout}s") from exc
if result.returncode != 0:
raise GitError(f"git {' '.join(args)} failed: {_one_line(result.stderr)}")
return result.stdout
def _one_line(text: str) -> str:
"""Squash git's multi-line stderr into one capped line.
A permanently unreachable source warns on every tick, so a six-line stderr would
put thousands of lines a day into the log and drown everything else.
"""
collapsed = " ".join(text.split())
return collapsed[:GIT_ERROR_CHARS] + ("…" if len(collapsed) > GIT_ERROR_CHARS else "")
def remote_head(url: str) -> str:
"""Remote HEAD without fetching — the right tool for a per-minute cadence."""
output = _git(["ls-remote", "--symref", url, "HEAD"], timeout=LS_REMOTE_TIMEOUT)
for line in output.splitlines():
parts = line.split()
if len(parts) == 2 and parts[1] == "HEAD":
return parts[0]
raise GitError(f"no HEAD in ls-remote output for {url}")
def _diff_names(root: Path, base_rev: str) -> tuple[list[str], list[str]] | None:
"""(changed, deleted) between base_rev and FETCH_HEAD, or None if base_rev is gone."""
try:
output = _git(["diff", "--name-status", f"{base_rev}..FETCH_HEAD"], cwd=root)
except GitError:
return None
changed, deleted = [], []
for line in output.splitlines():
fields = line.split("\t")
if len(fields) < 2:
continue
status = fields[0]
if status.startswith("R") and len(fields) >= 3:
deleted.append(fields[1])
changed.append(fields[2])
elif status.startswith("D"):
deleted.append(fields[1])
else:
changed.append(fields[1])
return changed, deleted
def plan_git_source(conn, source: SourceConfig, root: Path, full: bool) -> SourcePlan:
rev = remote_head(str(source.url))
row = store.get_source(conn, source.source_id)
indexed_rev = row["indexed_rev"] if row else None
if not root.exists():
_git(["clone", str(source.url), str(root)])
indexed_rev = None
if not full and indexed_rev == rev:
return SourcePlan(new_rev=rev)
_git(["fetch", "--prune"], cwd=root)
diff = None if (full or not indexed_rev) else _diff_names(root, indexed_rev)
_git(["reset", "--hard", "FETCH_HEAD"], cwd=root)
indexed = store.list_source_files(conn, source.source_id)
on_disk = walk_source(source, root)
if diff is None:
# Fresh clone, --full, or an indexed_rev the repo no longer has: reindex it all.
return SourcePlan(
changed=sorted(on_disk),
deleted=sorted(path for path in indexed if path not in on_disk),
new_rev=rev,
)
raw_changed, raw_deleted = diff
changed = sorted({path for path in raw_changed if path in on_disk})
deleted = sorted(
{path for path in raw_deleted if path in indexed} | {path for path in indexed if path not in on_disk}
)
return SourcePlan(changed=changed, deleted=[p for p in deleted if p not in changed], new_rev=rev)
# ---------------------------------------------------------------------------
# workspace driver
# ---------------------------------------------------------------------------
def plan_workspace_source(conn, source: SourceConfig, root: Path, full: bool) -> SourcePlan:
on_disk = walk_source(source, root)
indexed = store.list_source_files(conn, source.source_id)
plan = SourcePlan()
for rel_path, (size, mtime) in sorted(on_disk.items()):
row = indexed.get(rel_path)
if full or row is None:
plan.changed.append(rel_path)
continue
if row["size"] == size and row["mtime"] == mtime:
continue
if _sha256(root / rel_path) == row["sha256"]:
# Content is identical; refresh the stats so the next tick stays cheap.
plan.stat_refresh.append((rel_path, size, mtime))
else:
plan.changed.append(rel_path)
plan.deleted = sorted(path for path in indexed if path not in on_disk)
return plan
# ---------------------------------------------------------------------------
# indexing
# ---------------------------------------------------------------------------
def index_file(conn, source_id: str, root: Path, rel_path: str, now: str) -> int:
"""Re-chunk one file. Returns the chunk count, or -1 when it could not be read."""
file_path = root / rel_path
try:
text = file_path.read_text(encoding="utf-8")
stat = file_path.stat()
except (OSError, UnicodeDecodeError):
return -1
parsed = parse_markdown(text, rel_path)
store.upsert_file(
conn,
source_id,
rel_path,
parsed.title,
parsed.tags,
parsed.headings,
_sha256(file_path),
stat.st_size,
stat.st_mtime,
now,
)
store.replace_chunks(conn, source_id, rel_path, [(c.breadcrumb, c.text) for c in parsed.chunks])
return len(parsed.chunks)
def apply_plan(conn, source: SourceConfig, root: Path, plan: SourcePlan) -> dict[str, int]:
counts = {"indexed": 0, "deleted": 0, "unreadable": 0, "chunks": 0}
now = _now()
for rel_path in plan.deleted:
with store.tx(conn):
store.delete_file(conn, source.source_id, rel_path)
counts["deleted"] += 1
for rel_path in plan.changed:
with store.tx(conn):
chunks = index_file(conn, source.source_id, root, rel_path, now)
if chunks < 0:
counts["unreadable"] += 1
log(f"WARN {source.source_id}: unreadable {rel_path}")
continue
counts["indexed"] += 1
counts["chunks"] += chunks
if plan.stat_refresh:
with store.tx(conn):
for rel_path, size, mtime in plan.stat_refresh:
store.touch_file_stat(conn, source.source_id, rel_path, size, mtime)
with store.tx(conn):
store.mark_synced(conn, source.source_id, plan.new_rev, now)
return counts
def drain_pending(conn, config: WikiConfig) -> tuple[int, str | None]:
"""Embed every chunk still missing a vector. Returns (embedded, warning)."""
if not store.has_pending(conn):
return 0, None
embedder = OllamaEmbedder(config.embedding)
embedded = 0
while True:
batch = store.pending_chunks(conn, config.embedding.batch)
if not batch:
return embedded, None
try:
vectors = embedder.embed_documents([row["text"] for row in batch])
except EmbeddingUnavailable as exc:
return embedded, f"embeddings unavailable, staying FTS-only: {exc}"
now = _now()
with store.tx(conn):
for row, vector in zip(batch, vectors, strict=True):
store.store_embedding(conn, row["id"], vector, now)
embedded += len(batch)
# ---------------------------------------------------------------------------
# run
# ---------------------------------------------------------------------------
def ensure_meta(conn, config: WikiConfig, full: bool) -> str | None:
"""Write the index identity, or report a mismatch that only --full can resolve."""
stored = store.read_meta(conn)
mismatches = meta_mismatches(stored, config.embedding)
if mismatches and not full:
return f"index identity mismatch on {', '.join(mismatches)} — run wiki_sync.py --full"
with store.tx(conn):
store.write_meta(conn, expected_meta(config.embedding))
return None
def run(config: WikiConfig, args: argparse.Namespace) -> int:
sources = [s for s in config.sources if not args.source or s.source_id == args.source]
if not sources:
log(f"WARN unknown source {args.source!r}")
return 1
with store.connection(db_path()) as conn:
problem = ensure_meta(conn, config, args.full)
if problem:
log(f"WARN {problem}")
return 1
for source in sources:
with store.tx(conn):
store.upsert_source(conn, source.source_id, source.kind)
warnings: list[str] = []
did_work = args.full
for source in sources:
root = source_root(source)
try:
if source.kind == GIT_KIND:
plan = plan_git_source(conn, source, root, args.full)
else:
plan = plan_workspace_source(conn, source, root, args.full)
except GitError as exc:
# indexed_rev stays untouched, so the next tick retries this source.
warnings.append(f"{source.source_id}: {exc}")
log(f"WARN {source.source_id}: {exc}")
continue
row = store.get_source(conn, source.source_id)
indexed_rev = row["indexed_rev"] if row else None
rev_moved = plan.new_rev is not None and plan.new_rev != indexed_rev
if not plan.has_work and not rev_moved:
continue
did_work = True
if not plan.has_work:
# Upstream moved but touched nothing we index — record the rev so the
# next tick stops fetching.
with store.tx(conn):
store.mark_synced(conn, source.source_id, plan.new_rev, _now())
log(f"{source.source_id}: rev moved to {plan.new_rev}, no indexed file changed")
continue
counts = apply_plan(conn, source, root, plan)
log(
f"{source.source_id}: indexed {counts['indexed']} files "
f"({counts['chunks']} chunks), deleted {counts['deleted']}"
)
embedded, embed_warning = drain_pending(conn, config)
if embed_warning:
warnings.append(embed_warning)
log(f"WARN {embed_warning}")
if not did_work and not embedded and not warnings:
return 0
for source in sources:
uncovered = uncovered_top_level_dirs(source, source_root(source))
if uncovered:
log(f"coverage {source.source_id}: markdown outside paths in {', '.join(uncovered)}")
stats = store.index_stats(conn)
log(
f"done files={stats['files']} chunks={stats['chunks']} vectors={stats['vectors']} "
f"pending={stats['pending']} embedded={embedded}"
)
return 1 if warnings else 0
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Index the wiki sources into wiki/index.sqlite.")
parser.add_argument(
"--full",
action="store_true",
help="wipe and rebuild the index (needed after a model or chunker change)",
)
parser.add_argument("--source", help="limit the run to one source id")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
try:
config = load_config()
except ConfigError as exc:
log(f"WARN {exc}")
print(f"config error: {exc}", file=sys.stderr)
return 1
WIKI_DIR.mkdir(parents=True, exist_ok=True)
if not acquire_lock():
return 0
try:
if args.full:
log("full reindex requested")
with store.transaction(db_path()) as conn:
store.reset_index(conn)
return run(config, args)
finally:
release_lock()
if __name__ == "__main__":
sys.exit(main())