provozni zaloha
This commit is contained in:
lachtan
541
skills/llm-wiki/scripts/wiki_graph_extract.py
Normal file
541
skills/llm-wiki/scripts/wiki_graph_extract.py
Normal file
@@ -0,0 +1,541 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = ["pyyaml"]
|
||||
# ///
|
||||
"""
|
||||
wiki_graph_extract.py — Compile the markdown wiki into a queryable graph.
|
||||
|
||||
Markdown remains canonical. This script reads every wiki page, derives nodes
|
||||
and edges (typed semantic edges from `graph.relationships`, plus implicit
|
||||
`mentions`, `sourced_from`, `summarizes_raw` edges), and emits artifacts under
|
||||
`<wiki>/graph/` that can be deleted and rebuilt at any time.
|
||||
|
||||
Requires PyYAML (`pip install pyyaml`) — the new graph layer uses real YAML
|
||||
parsing for its nested frontmatter, unlike the stdlib-only lint/search/stats
|
||||
scripts.
|
||||
|
||||
Usage:
|
||||
python wiki_graph_extract.py <wiki-dir> [options]
|
||||
|
||||
Options:
|
||||
--out <dir> Output directory (default: <wiki-dir>/graph)
|
||||
--formats jsonl,sqlite,... Comma-list of formats to emit
|
||||
(jsonl, sqlite, graphml; default: all three)
|
||||
--ontology <path> Override ontology path
|
||||
(default: <wiki-dir>/graph/ontology.yaml)
|
||||
|
||||
Examples:
|
||||
python wiki_graph_extract.py wiki/
|
||||
python wiki_graph_extract.py wiki/ --out wiki/graph --formats jsonl,sqlite
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print(
|
||||
"wiki_graph_extract.py requires PyYAML.\n"
|
||||
"Install with: pip install pyyaml",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
|
||||
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL)
|
||||
|
||||
SKIP_TOP_LEVEL_FILES = {"SCHEMA.md", "index.md", "log.md", "README.md"}
|
||||
SKIP_TOP_LEVEL_DIRS = {"indexes", "graph"}
|
||||
|
||||
DEFAULT_FORMATS = ["jsonl", "sqlite", "graphml"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Page collection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict, str]:
|
||||
"""Extract YAML frontmatter using PyYAML. Returns (meta, body)."""
|
||||
m = FRONTMATTER_RE.match(text)
|
||||
if not m:
|
||||
return {}, text
|
||||
fm_text = m.group(1)
|
||||
body = text[m.end():]
|
||||
try:
|
||||
meta = yaml.safe_load(fm_text) or {}
|
||||
except yaml.YAMLError:
|
||||
meta = {}
|
||||
if not isinstance(meta, dict):
|
||||
meta = {}
|
||||
return meta, body
|
||||
|
||||
|
||||
def collect_pages(wiki_root: Path) -> list[dict]:
|
||||
pages = []
|
||||
for md_path in sorted(wiki_root.rglob("*.md")):
|
||||
rel = md_path.relative_to(wiki_root)
|
||||
if rel.parts[0] in SKIP_TOP_LEVEL_FILES or rel.parts[0] in SKIP_TOP_LEVEL_DIRS:
|
||||
continue
|
||||
if rel.name.startswith("."):
|
||||
continue
|
||||
try:
|
||||
text = md_path.read_text(encoding="utf-8")
|
||||
except (UnicodeDecodeError, OSError):
|
||||
continue
|
||||
meta, body = parse_frontmatter(text)
|
||||
links = [m.group(1).strip() for m in WIKILINK_RE.finditer(body)]
|
||||
pages.append({
|
||||
"path": str(md_path),
|
||||
"rel_path": str(rel).replace("\\", "/"),
|
||||
"slug": md_path.stem,
|
||||
"meta": meta,
|
||||
"body": body,
|
||||
"links": links,
|
||||
})
|
||||
return pages
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ontology
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_ontology(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {"node_types": {}, "predicates": {}}
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
except yaml.YAMLError as e:
|
||||
print(f"Ontology parse error ({path}): {e}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
data.setdefault("node_types", {})
|
||||
data.setdefault("predicates", {})
|
||||
return data
|
||||
|
||||
|
||||
def derive_node_type(meta: dict, ontology: dict) -> str | None:
|
||||
"""Map a page's frontmatter to a node_type using ontology[node_types][*].maps_from."""
|
||||
page_type = meta.get("type")
|
||||
page_kind = meta.get("kind")
|
||||
explicit = (meta.get("graph") or {}).get("node_type") if isinstance(meta.get("graph"), dict) else None
|
||||
if explicit:
|
||||
return explicit
|
||||
# Try (type, kind) match first, then type-only.
|
||||
type_kind_match = None
|
||||
type_only_match = None
|
||||
for nt_name, nt_def in ontology["node_types"].items():
|
||||
maps = (nt_def or {}).get("maps_from") or {}
|
||||
m_type = maps.get("type")
|
||||
m_kind = maps.get("kind")
|
||||
if m_type and m_type == page_type:
|
||||
if m_kind and m_kind == page_kind:
|
||||
type_kind_match = nt_name
|
||||
break
|
||||
if not m_kind and type_only_match is None:
|
||||
type_only_match = nt_name
|
||||
return type_kind_match or type_only_match
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Node + edge construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_nodes(pages: list[dict], ontology: dict) -> tuple[list[dict], dict, list[dict]]:
|
||||
"""Build the node list + slug→node_id index + alias rows. Returns (nodes, slug_to_id, aliases)."""
|
||||
nodes: list[dict] = []
|
||||
slug_to_id: dict[str, str] = {}
|
||||
aliases: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
for p in pages:
|
||||
meta = p["meta"]
|
||||
graph_meta = meta.get("graph") if isinstance(meta.get("graph"), dict) else {}
|
||||
node_type = derive_node_type(meta, ontology) or "concept"
|
||||
explicit_id = graph_meta.get("node_id")
|
||||
node_id = explicit_id or f"{node_type}:{p['slug']}"
|
||||
|
||||
# Skip duplicates — first one wins; lint will flag this.
|
||||
if node_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(node_id)
|
||||
|
||||
node = {
|
||||
"id": node_id,
|
||||
"slug": p["slug"],
|
||||
"title": meta.get("title") or p["slug"],
|
||||
"page_type": meta.get("type") or "",
|
||||
"node_type": node_type,
|
||||
"kind": meta.get("kind") or "",
|
||||
"tags": list(meta.get("tags") or []),
|
||||
"aliases": list(graph_meta.get("aliases") or []),
|
||||
"path": p["rel_path"],
|
||||
"created": meta.get("created") or "",
|
||||
"updated": meta.get("updated") or "",
|
||||
"canonical": bool(graph_meta.get("canonical", False)),
|
||||
}
|
||||
nodes.append(node)
|
||||
slug_to_id[p["slug"]] = node_id
|
||||
for alias in node["aliases"]:
|
||||
aliases.append({"alias": str(alias), "node_id": node_id})
|
||||
|
||||
return nodes, slug_to_id, aliases
|
||||
|
||||
|
||||
def edge_id(subject: str, predicate: str, obj: str, source: str | None, evidence: str | None) -> str:
|
||||
# Truncated to 96 bits — collision risk is negligible at any plausible
|
||||
# wiki scale and shorter ids keep the JSONL/sqlite/graphml outputs readable.
|
||||
h = hashlib.sha256()
|
||||
parts = [subject or "", predicate or "", obj or "", source or "", evidence or ""]
|
||||
h.update("\x1f".join(parts).encode("utf-8"))
|
||||
return h.hexdigest()[:24]
|
||||
|
||||
|
||||
def make_edge(*, subject, predicate, obj, source, evidence, confidence, status,
|
||||
extraction_method, page, extras: dict | None = None) -> dict:
|
||||
return {
|
||||
"id": edge_id(subject, predicate, obj, source, evidence),
|
||||
"subject": subject,
|
||||
"predicate": predicate,
|
||||
"object": obj,
|
||||
"source": source or "",
|
||||
"evidence": evidence or "",
|
||||
"confidence": confidence or "",
|
||||
"status": status or "",
|
||||
"extraction_method": extraction_method,
|
||||
"page": page,
|
||||
"extras": extras or {},
|
||||
}
|
||||
|
||||
|
||||
def build_edges(pages: list[dict], slug_to_id: dict[str, str]) -> list[dict]:
|
||||
edges: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
def push(edge: dict) -> None:
|
||||
if edge["id"] in seen_ids:
|
||||
return
|
||||
seen_ids.add(edge["id"])
|
||||
edges.append(edge)
|
||||
|
||||
for p in pages:
|
||||
slug = p["slug"]
|
||||
subject_id = slug_to_id.get(slug)
|
||||
if not subject_id:
|
||||
continue
|
||||
meta = p["meta"]
|
||||
graph_meta = meta.get("graph") if isinstance(meta.get("graph"), dict) else {}
|
||||
|
||||
# 1. Typed semantic edges from graph.relationships[].
|
||||
for rel in graph_meta.get("relationships") or []:
|
||||
if not isinstance(rel, dict):
|
||||
continue
|
||||
obj = rel.get("object")
|
||||
predicate = rel.get("predicate")
|
||||
if not (obj and predicate):
|
||||
continue
|
||||
extras = {
|
||||
k: rel[k] for k in ("valid_from", "valid_to", "notes", "raw_ref",
|
||||
"contradicts", "supersedes")
|
||||
if k in rel and rel[k] is not None
|
||||
}
|
||||
push(make_edge(
|
||||
subject=subject_id,
|
||||
predicate=str(predicate),
|
||||
obj=str(obj),
|
||||
source=rel.get("source"),
|
||||
evidence=rel.get("evidence"),
|
||||
confidence=rel.get("confidence"),
|
||||
status=rel.get("status"),
|
||||
extraction_method="explicit_graph_frontmatter",
|
||||
page=p["rel_path"],
|
||||
extras=extras,
|
||||
))
|
||||
|
||||
# 2. Mentions edges from body wikilinks.
|
||||
seen_targets: set[str] = set()
|
||||
for link in p["links"]:
|
||||
target_slug = link.split("#")[0].strip()
|
||||
if not target_slug or target_slug == slug:
|
||||
continue
|
||||
target_id = slug_to_id.get(target_slug)
|
||||
if not target_id or target_id in seen_targets:
|
||||
continue
|
||||
seen_targets.add(target_id)
|
||||
push(make_edge(
|
||||
subject=subject_id,
|
||||
predicate="mentions",
|
||||
obj=target_id,
|
||||
source=None,
|
||||
evidence=None,
|
||||
confidence="low",
|
||||
status="current",
|
||||
extraction_method="body_wikilink",
|
||||
page=p["rel_path"],
|
||||
))
|
||||
|
||||
# 3. sourced_from edges from frontmatter `sources:` (skip on source pages themselves).
|
||||
if meta.get("type") != "source":
|
||||
for src_slug in meta.get("sources") or []:
|
||||
src_id = slug_to_id.get(str(src_slug))
|
||||
if not src_id:
|
||||
continue
|
||||
push(make_edge(
|
||||
subject=subject_id,
|
||||
predicate="sourced_from",
|
||||
obj=src_id,
|
||||
source=str(src_slug),
|
||||
evidence=None,
|
||||
confidence="high",
|
||||
status="current",
|
||||
extraction_method="frontmatter_sources",
|
||||
page=p["rel_path"],
|
||||
))
|
||||
|
||||
# 4. summarizes_raw edges from source pages' raw: field.
|
||||
if meta.get("type") == "source":
|
||||
raw_path = meta.get("raw")
|
||||
if raw_path:
|
||||
push(make_edge(
|
||||
subject=subject_id,
|
||||
predicate="summarizes_raw",
|
||||
obj=f"raw:{raw_path}",
|
||||
source=None,
|
||||
evidence=None,
|
||||
confidence="high",
|
||||
status="current",
|
||||
extraction_method="frontmatter_raw",
|
||||
page=p["rel_path"],
|
||||
))
|
||||
|
||||
return edges
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output writers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _normalize_for_json(value):
|
||||
if hasattr(value, "isoformat"):
|
||||
return value.isoformat()
|
||||
if isinstance(value, list):
|
||||
return [_normalize_for_json(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: _normalize_for_json(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def write_jsonl(out_dir: Path, nodes: list[dict], edges: list[dict]) -> None:
|
||||
nodes_sorted = sorted(nodes, key=lambda n: n["id"])
|
||||
edges_sorted = sorted(edges, key=lambda e: e["id"])
|
||||
with (out_dir / "nodes.jsonl").open("w", encoding="utf-8") as f:
|
||||
for n in nodes_sorted:
|
||||
f.write(json.dumps(_normalize_for_json(n), sort_keys=True, ensure_ascii=False))
|
||||
f.write("\n")
|
||||
with (out_dir / "edges.jsonl").open("w", encoding="utf-8") as f:
|
||||
for e in edges_sorted:
|
||||
f.write(json.dumps(_normalize_for_json(e), sort_keys=True, ensure_ascii=False))
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def write_sqlite(out_dir: Path, nodes: list[dict], aliases: list[dict], edges: list[dict]) -> None:
|
||||
db_path = out_dir / "graph.sqlite"
|
||||
if db_path.exists():
|
||||
db_path.unlink()
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.executescript("""
|
||||
CREATE TABLE nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
page_type TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
kind TEXT,
|
||||
path TEXT NOT NULL,
|
||||
created TEXT,
|
||||
updated TEXT,
|
||||
metadata_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE aliases (
|
||||
alias TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
PRIMARY KEY (alias, node_id),
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id)
|
||||
);
|
||||
CREATE TABLE edges (
|
||||
id TEXT PRIMARY KEY,
|
||||
subject TEXT NOT NULL,
|
||||
predicate TEXT NOT NULL,
|
||||
object TEXT NOT NULL,
|
||||
source TEXT,
|
||||
evidence TEXT,
|
||||
confidence TEXT,
|
||||
status TEXT,
|
||||
extraction_method TEXT NOT NULL,
|
||||
page TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_edges_subject ON edges(subject);
|
||||
CREATE INDEX idx_edges_object ON edges(object);
|
||||
CREATE INDEX idx_edges_predicate ON edges(predicate);
|
||||
CREATE INDEX idx_edges_source ON edges(source);
|
||||
""")
|
||||
|
||||
for n in sorted(nodes, key=lambda n: n["id"]):
|
||||
metadata_json = json.dumps(_normalize_for_json({
|
||||
"tags": n.get("tags", []),
|
||||
"aliases": n.get("aliases", []),
|
||||
"canonical": n.get("canonical", False),
|
||||
}), sort_keys=True, ensure_ascii=False)
|
||||
conn.execute(
|
||||
"INSERT INTO nodes (id, slug, title, page_type, node_type, kind, path, created, updated, metadata_json) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
n["id"], n["slug"], n["title"], n["page_type"], n["node_type"],
|
||||
n.get("kind") or None, n["path"],
|
||||
str(n.get("created") or "") or None,
|
||||
str(n.get("updated") or "") or None,
|
||||
metadata_json,
|
||||
),
|
||||
)
|
||||
|
||||
for a in sorted(aliases, key=lambda a: (a["alias"], a["node_id"])):
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO aliases (alias, node_id) VALUES (?, ?)",
|
||||
(a["alias"], a["node_id"]),
|
||||
)
|
||||
|
||||
for e in sorted(edges, key=lambda e: e["id"]):
|
||||
metadata_json = json.dumps(_normalize_for_json(e.get("extras") or {}),
|
||||
sort_keys=True, ensure_ascii=False)
|
||||
conn.execute(
|
||||
"INSERT INTO edges (id, subject, predicate, object, source, evidence, "
|
||||
"confidence, status, extraction_method, page, metadata_json) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
e["id"], e["subject"], e["predicate"], e["object"],
|
||||
e.get("source") or None, e.get("evidence") or None,
|
||||
e.get("confidence") or None, e.get("status") or None,
|
||||
e["extraction_method"], e["page"], metadata_json,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def write_graphml(out_dir: Path, nodes: list[dict], edges: list[dict]) -> None:
|
||||
ns = "http://graphml.graphdrawing.org/xmlns"
|
||||
ET.register_namespace("", ns)
|
||||
root = ET.Element(f"{{{ns}}}graphml")
|
||||
|
||||
keys = [
|
||||
("d_title", "node", "title", "string"),
|
||||
("d_node_type", "node", "node_type", "string"),
|
||||
("d_page_type", "node", "page_type", "string"),
|
||||
("d_path", "node", "path", "string"),
|
||||
("d_predicate", "edge", "predicate", "string"),
|
||||
("d_confidence", "edge", "confidence", "string"),
|
||||
("d_status", "edge", "status", "string"),
|
||||
("d_source", "edge", "source", "string"),
|
||||
]
|
||||
for kid, kfor, kname, ktype in keys:
|
||||
k = ET.SubElement(root, f"{{{ns}}}key")
|
||||
k.set("id", kid)
|
||||
k.set("for", kfor)
|
||||
k.set("attr.name", kname)
|
||||
k.set("attr.type", ktype)
|
||||
|
||||
graph = ET.SubElement(root, f"{{{ns}}}graph")
|
||||
graph.set("id", "wiki")
|
||||
graph.set("edgedefault", "directed")
|
||||
|
||||
for n in sorted(nodes, key=lambda n: n["id"]):
|
||||
node_el = ET.SubElement(graph, f"{{{ns}}}node")
|
||||
node_el.set("id", n["id"])
|
||||
for kid, kfor, kname, _ in keys:
|
||||
if kfor != "node":
|
||||
continue
|
||||
data = ET.SubElement(node_el, f"{{{ns}}}data")
|
||||
data.set("key", kid)
|
||||
data.text = str(n.get(kname) or "")
|
||||
|
||||
for e in sorted(edges, key=lambda e: e["id"]):
|
||||
edge_el = ET.SubElement(graph, f"{{{ns}}}edge")
|
||||
edge_el.set("id", e["id"])
|
||||
edge_el.set("source", e["subject"])
|
||||
edge_el.set("target", e["object"])
|
||||
for kid, kfor, kname, _ in keys:
|
||||
if kfor != "edge":
|
||||
continue
|
||||
data = ET.SubElement(edge_el, f"{{{ns}}}data")
|
||||
data.set("key", kid)
|
||||
data.text = str(e.get(kname) or "")
|
||||
|
||||
tree = ET.ElementTree(root)
|
||||
ET.indent(tree, space=" ")
|
||||
tree.write(out_dir / "graph.graphml", encoding="utf-8", xml_declaration=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("wiki", type=Path, help="Wiki directory.")
|
||||
parser.add_argument("--out", type=Path, help="Output directory (default: <wiki>/graph)")
|
||||
parser.add_argument("--formats", default=",".join(DEFAULT_FORMATS),
|
||||
help="Comma-list: jsonl, sqlite, graphml")
|
||||
parser.add_argument("--ontology", type=Path, help="Ontology file (default: <wiki>/graph/ontology.yaml)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.wiki.exists():
|
||||
print(f"Wiki directory not found: {args.wiki}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
out_dir = args.out or (args.wiki / "graph")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
ontology_path = args.ontology or (args.wiki / "graph" / "ontology.yaml")
|
||||
ontology = load_ontology(ontology_path)
|
||||
formats = [f.strip().lower() for f in args.formats.split(",") if f.strip()]
|
||||
unknown = [f for f in formats if f not in DEFAULT_FORMATS]
|
||||
if unknown:
|
||||
print(f"Unknown formats: {unknown}. Allowed: {DEFAULT_FORMATS}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
pages = collect_pages(args.wiki)
|
||||
nodes, slug_to_id, aliases = build_nodes(pages, ontology)
|
||||
edges = build_edges(pages, slug_to_id)
|
||||
|
||||
if "jsonl" in formats:
|
||||
write_jsonl(out_dir, nodes, edges)
|
||||
if "sqlite" in formats:
|
||||
write_sqlite(out_dir, nodes, aliases, edges)
|
||||
if "graphml" in formats:
|
||||
write_graphml(out_dir, nodes, edges)
|
||||
|
||||
print(f"Extracted {len(nodes)} nodes, {len(edges)} edges → {out_dir}")
|
||||
breakdown = defaultdict(int)
|
||||
for e in edges:
|
||||
breakdown[e["predicate"]] += 1
|
||||
for pred, count in sorted(breakdown.items(), key=lambda x: (-x[1], x[0])):
|
||||
print(f" {pred:20s} {count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user