267 lines
9.6 KiB
Python
267 lines
9.6 KiB
Python
"""Git driver: clone, ls-remote change detection, diff, deletions, per-source failure."""
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
SCRIPTS = Path(__file__).parent.parent / "scripts"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
import wiki_store as store
|
|
import wiki_sync
|
|
from conftest import WORKSPACE_SOURCE
|
|
|
|
DOC = """# Zálohování
|
|
|
|
Záloha běží každou noc přes rsync na druhý disk.
|
|
"""
|
|
|
|
TRAVEL_DOC = """# Tokijské metro
|
|
|
|
Z Narity do centra jede Skyliner za osmatřicet minut.
|
|
"""
|
|
|
|
GIT_SOURCE = """ notes:
|
|
kind: git
|
|
url: {url}
|
|
paths: ["**"]
|
|
include: ["*.md"]
|
|
exclude:
|
|
- "**/node_modules/**"
|
|
- "**/vendor/**"
|
|
"""
|
|
|
|
|
|
def _git(args, cwd=None):
|
|
result = subprocess.run(
|
|
["git", "-c", "user.email=t@t", "-c", "user.name=t", *args],
|
|
cwd=cwd,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
return result.stdout
|
|
|
|
|
|
class Remote:
|
|
"""A bare repo plus a working clone to push commits from."""
|
|
|
|
def __init__(self, root: Path):
|
|
self.bare = root / "origin.git"
|
|
self.work = root / "origin-work"
|
|
_git(["init", "--bare", "--initial-branch=master", str(self.bare)])
|
|
_git(["clone", str(self.bare), str(self.work)])
|
|
|
|
@property
|
|
def url(self) -> str:
|
|
return str(self.bare)
|
|
|
|
def commit(self, files: dict[str, str | None], message: str = "change") -> str:
|
|
for rel_path, text in files.items():
|
|
path = self.work / rel_path
|
|
if text is None:
|
|
_git(["rm", "-q", rel_path], cwd=self.work)
|
|
continue
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text, encoding="utf-8")
|
|
_git(["add", rel_path], cwd=self.work)
|
|
_git(["commit", "-m", message], cwd=self.work)
|
|
_git(["push", "-q", "origin", "master"], cwd=self.work)
|
|
return _git(["rev-parse", "HEAD"], cwd=self.work).strip()
|
|
|
|
def move(self, old: str, new: str) -> str:
|
|
(self.work / new).parent.mkdir(parents=True, exist_ok=True)
|
|
_git(["mv", old, new], cwd=self.work)
|
|
_git(["commit", "-m", "rename"], cwd=self.work)
|
|
_git(["push", "-q", "origin", "master"], cwd=self.work)
|
|
return _git(["rev-parse", "HEAD"], cwd=self.work).strip()
|
|
|
|
|
|
def _use_git_source(env, url, with_workspace=False):
|
|
sources = GIT_SOURCE.format(url=url)
|
|
if with_workspace:
|
|
sources += WORKSPACE_SOURCE
|
|
env.write_config(sources)
|
|
|
|
|
|
def _stats(env):
|
|
with store.connection(env.db_path) as conn:
|
|
return store.index_stats(conn)
|
|
|
|
|
|
def test_first_run_clones_and_indexes(wiki_env, fake_embedder, tmp_path):
|
|
remote = Remote(tmp_path)
|
|
head = remote.commit({"zalohy.md": DOC, "japan/metro.md": TRAVEL_DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
clone = wiki_env.wiki_dir / "remote" / "notes"
|
|
assert (clone / "zalohy.md").is_file() # non-bare: a working tree grep can read
|
|
assert (clone / ".git").is_dir()
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert sorted(store.list_source_files(conn, "notes")) == ["japan/metro.md", "zalohy.md"]
|
|
source = store.get_source(conn, "notes")
|
|
assert source is not None
|
|
assert source["indexed_rev"] == head
|
|
assert source["kind"] == "git"
|
|
stats = _stats(wiki_env)
|
|
assert stats["vectors"] == stats["chunks"] > 0
|
|
|
|
|
|
def test_second_run_over_the_same_revision_changes_nothing(wiki_env, fake_embedder, tmp_path):
|
|
"""Verification 2 for the git driver."""
|
|
remote = Remote(tmp_path)
|
|
remote.commit({"zalohy.md": DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
before = _stats(wiki_env)
|
|
log_before = wiki_env.log_text()
|
|
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
assert _stats(wiki_env) == before
|
|
assert wiki_env.log_text() == log_before # ls-remote matched, so not even a fetch
|
|
|
|
|
|
def test_new_commit_reindexes_only_the_changed_file(wiki_env, fake_embedder, tmp_path):
|
|
remote = Remote(tmp_path)
|
|
remote.commit({"zalohy.md": DOC, "japan/metro.md": TRAVEL_DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
metro_indexed_at = store.list_source_files(conn, "notes")["japan/metro.md"]["indexed_at"]
|
|
|
|
head = remote.commit({"zalohy.md": DOC + "\n## Offsite\n\nKopie jede do S3.\n"})
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
files = store.list_source_files(conn, "notes")
|
|
assert files["japan/metro.md"]["indexed_at"] == metro_indexed_at # untouched
|
|
source = store.get_source(conn, "notes")
|
|
assert source is not None and source["indexed_rev"] == head
|
|
assert len(store.bm25_ranked_ids(conn, "offsite", 10)) == 1
|
|
assert store.orphan_vector_ids(conn) == []
|
|
|
|
|
|
def test_deleted_file_upstream_drops_its_chunks_and_vectors(wiki_env, fake_embedder, tmp_path):
|
|
remote = Remote(tmp_path)
|
|
remote.commit({"zalohy.md": DOC, "japan/metro.md": TRAVEL_DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
remote.commit({"japan/metro.md": None}, message="drop metro")
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert sorted(store.list_source_files(conn, "notes")) == ["zalohy.md"]
|
|
assert store.bm25_ranked_ids(conn, "skyliner", 10) == []
|
|
assert store.orphan_vector_ids(conn) == []
|
|
stats = _stats(wiki_env)
|
|
assert stats["vectors"] == stats["chunks"]
|
|
|
|
|
|
def test_renamed_file_moves_its_chunks(wiki_env, fake_embedder, tmp_path):
|
|
remote = Remote(tmp_path)
|
|
remote.commit({"japan/metro.md": TRAVEL_DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
remote.move("japan/metro.md", "japan/tokyo-metro.md")
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert sorted(store.list_source_files(conn, "notes")) == ["japan/tokyo-metro.md"]
|
|
assert len(store.bm25_ranked_ids(conn, "skyliner", 10)) == 1
|
|
assert store.orphan_vector_ids(conn) == []
|
|
|
|
|
|
def test_commit_touching_nothing_indexed_only_records_the_rev(wiki_env, fake_embedder, tmp_path):
|
|
remote = Remote(tmp_path)
|
|
remote.commit({"zalohy.md": DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
before = _stats(wiki_env)
|
|
|
|
head = remote.commit({"tool.py": "print('hello')\n"})
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
assert _stats(wiki_env) == before
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
source = store.get_source(conn, "notes")
|
|
assert source is not None and source["indexed_rev"] == head
|
|
assert "no indexed file changed" in wiki_env.log_text()
|
|
|
|
|
|
def test_vendored_markdown_is_excluded(wiki_env, fake_embedder, tmp_path):
|
|
remote = Remote(tmp_path)
|
|
remote.commit(
|
|
{
|
|
"zalohy.md": DOC,
|
|
"node_modules/pkg/README.md": "# Foreign readme\n\nnenasazovat\n",
|
|
"web/vendor/lib/CHANGELOG.md": "# Changelog\n\nnenasazovat\n",
|
|
}
|
|
)
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert sorted(store.list_source_files(conn, "notes")) == ["zalohy.md"]
|
|
|
|
|
|
def test_unreachable_remote_warns_skips_and_lets_other_sources_finish(wiki_env, fake_embedder, tmp_path):
|
|
"""Verification 8: per-source failure, indexed_rev untouched, workspace completes."""
|
|
_use_git_source(wiki_env, str(tmp_path / "does-not-exist.git"), with_workspace=True)
|
|
wiki_env.write_file("notes/local.md", DOC)
|
|
|
|
assert wiki_sync.main([]) == 1
|
|
|
|
log = wiki_env.log_text()
|
|
assert "WARN notes:" in log
|
|
# A permanent failure warns every tick, so each warning must stay a single line.
|
|
assert len([line for line in log.splitlines() if "WARN notes:" in line]) == 1
|
|
assert all(line.strip() for line in log.splitlines())
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert store.list_source_files(conn, "notes") == {}
|
|
git_source = store.get_source(conn, "notes")
|
|
assert git_source is not None and git_source["indexed_rev"] is None
|
|
# The workspace source, which has nothing to do with the network, finished.
|
|
assert sorted(store.list_source_files(conn, "workspace")) == ["notes/local.md"]
|
|
|
|
|
|
def test_remote_that_recovers_is_picked_up_on_the_next_tick(wiki_env, fake_embedder, tmp_path):
|
|
missing = tmp_path / "later.git"
|
|
_use_git_source(wiki_env, str(missing))
|
|
assert wiki_sync.main([]) == 1
|
|
|
|
remote = Remote(tmp_path / "real")
|
|
(tmp_path / "real").mkdir(exist_ok=True)
|
|
remote_bare = remote.bare
|
|
remote.commit({"zalohy.md": DOC})
|
|
_use_git_source(wiki_env, str(remote_bare))
|
|
|
|
assert wiki_sync.main([]) == 0
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert sorted(store.list_source_files(conn, "notes")) == ["zalohy.md"]
|
|
|
|
|
|
def test_missing_indexed_rev_falls_back_to_a_full_reindex(wiki_env, fake_embedder, tmp_path):
|
|
"""A force-pushed or gc'd base revision must degrade to a full pass, not crash."""
|
|
remote = Remote(tmp_path)
|
|
remote.commit({"zalohy.md": DOC})
|
|
_use_git_source(wiki_env, remote.url)
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn, store.tx(conn):
|
|
store.mark_synced(conn, "notes", "0" * 40, "2026-01-01T00:00:00+00:00")
|
|
|
|
remote.commit({"japan/metro.md": TRAVEL_DOC})
|
|
assert wiki_sync.main([]) == 0
|
|
|
|
with store.connection(wiki_env.db_path) as conn:
|
|
assert sorted(store.list_source_files(conn, "notes")) == ["japan/metro.md", "zalohy.md"]
|
|
assert store.orphan_vector_ids(conn) == []
|