160 lines
4.9 KiB
Python
160 lines
4.9 KiB
Python
#!/usr/bin/env -S uv run --script
|
|
# /// script
|
|
# requires-python = ">=3.11"
|
|
# dependencies = []
|
|
# ///
|
|
"""
|
|
wiki_stats.py — Quick summary of wiki size, shape, and link density.
|
|
|
|
Useful for deciding when to shard the index or split pages.
|
|
|
|
Usage:
|
|
python wiki_stats.py [<wiki-dir>]
|
|
|
|
Example:
|
|
python wiki_stats.py wiki/
|
|
"""
|
|
|
|
import argparse
|
|
import re
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
|
|
WIKILINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
|
|
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL)
|
|
|
|
|
|
SKIP_TOP_LEVEL_FILES = {"SCHEMA.md", "log.md", "README.md"}
|
|
SKIP_TOP_LEVEL_DIRS = {"indexes", "graph"}
|
|
|
|
|
|
def parse_type(text: str) -> str | None:
|
|
m = FRONTMATTER_RE.match(text)
|
|
if not m:
|
|
return None
|
|
fm = m.group(1)
|
|
for line in fm.split("\n"):
|
|
kv = re.match(r"^type:\s*(.*)$", line)
|
|
if kv:
|
|
return kv.group(1).strip().strip('"').strip("'")
|
|
return None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("wiki", nargs="?", type=Path, default=Path("cml/wiki"))
|
|
args = parser.parse_args()
|
|
|
|
if not args.wiki.exists():
|
|
print(f"Wiki directory not found: {args.wiki}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
total_pages = 0
|
|
total_lines = 0
|
|
total_words = 0
|
|
total_links = 0
|
|
pages_by_type = Counter()
|
|
pages_by_dir = Counter()
|
|
largest = []
|
|
most_linked_in = Counter()
|
|
index_lines = 0
|
|
|
|
for md_path in args.wiki.rglob("*.md"):
|
|
rel = md_path.relative_to(args.wiki)
|
|
try:
|
|
text = md_path.read_text(encoding="utf-8")
|
|
except (UnicodeDecodeError, OSError):
|
|
continue
|
|
|
|
if rel.name == "index.md" and len(rel.parts) == 1:
|
|
index_lines = text.count("\n") + 1
|
|
continue
|
|
if rel.parts[0] in SKIP_TOP_LEVEL_FILES:
|
|
continue
|
|
if rel.parts[0] in SKIP_TOP_LEVEL_DIRS:
|
|
continue
|
|
if rel.name.startswith("."):
|
|
continue
|
|
|
|
total_pages += 1
|
|
line_count = text.count("\n") + 1
|
|
word_count = len(text.split())
|
|
total_lines += line_count
|
|
total_words += word_count
|
|
# Strip frontmatter before counting wikilinks; frontmatter uses bare slugs.
|
|
body = FRONTMATTER_RE.sub("", text, count=1) if text.startswith("---") else text
|
|
links = WIKILINK_RE.findall(body)
|
|
total_links += len(links)
|
|
for link in links:
|
|
target = link.split("|")[0].strip()
|
|
most_linked_in[target] += 1
|
|
page_type = parse_type(text) or "(none)"
|
|
pages_by_type[page_type] += 1
|
|
if len(rel.parts) > 1:
|
|
pages_by_dir[rel.parts[0]] += 1
|
|
else:
|
|
pages_by_dir["(root)"] += 1
|
|
largest.append((line_count, str(rel)))
|
|
|
|
largest.sort(reverse=True)
|
|
|
|
print("=" * 60)
|
|
print(f"Wiki Stats: {args.wiki}")
|
|
print("=" * 60)
|
|
print(f"Pages: {total_pages}")
|
|
print(f"Total lines: {total_lines:,}")
|
|
print(f"Total words: {total_words:,}")
|
|
print(f"Total links: {total_links:,}")
|
|
if total_pages:
|
|
print(f"Avg page: {total_lines // total_pages} lines / {total_words // total_pages} words")
|
|
print(f"Link density: {total_links / total_pages:.1f} links per page")
|
|
print(f"index.md: {index_lines} lines" + (" ← shard recommended (>300)" if index_lines > 300 else ""))
|
|
print()
|
|
|
|
print("Pages by type:")
|
|
for t, n in pages_by_type.most_common():
|
|
print(f" {t:15s} {n}")
|
|
print()
|
|
|
|
print("Pages by directory:")
|
|
for d, n in pages_by_dir.most_common():
|
|
print(f" {d:15s} {n}")
|
|
print()
|
|
|
|
if largest:
|
|
print("Largest pages:")
|
|
for lines, path in largest[:10]:
|
|
warn = ""
|
|
if lines > 800:
|
|
warn = " ← OVER HARD CAP"
|
|
elif lines > 400:
|
|
warn = " ← over soft cap"
|
|
print(f" {lines:5d} {path}{warn}")
|
|
print()
|
|
|
|
if most_linked_in:
|
|
print("Most-linked-to pages (hubs):")
|
|
for slug, count in most_linked_in.most_common(10):
|
|
print(f" {count:4d} [[{slug}]]")
|
|
print()
|
|
|
|
# Scaling recommendations
|
|
print("Scaling thresholds:")
|
|
if total_pages < 50:
|
|
print(" → Below first threshold. Flat structure is fine.")
|
|
elif total_pages < 150 and index_lines < 300:
|
|
print(" → Below shard threshold. Continue with single index.md.")
|
|
elif (total_pages >= 150 or index_lines >= 300) and not (args.wiki / "indexes").exists():
|
|
print(" → AT SHARD THRESHOLD. Consider sharding index.md into wiki/indexes/<type>.md.")
|
|
print(" See references/scaling-playbook.md.")
|
|
elif total_pages >= 300:
|
|
print(" → Past 300 pages. Use scripts/wiki_search.py as a routine fallback.")
|
|
if total_pages >= 500:
|
|
print(" → Past 500 pages. Run lint weekly or per-N-ingests.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|