"""Git driver: clone, ls-remote change detection, diff, deletions, per-source failure.""" import subprocess import sys from pathlib import Path SCRIPTS = Path(__file__).parent.parent / "scripts" sys.path.insert(0, str(SCRIPTS)) import wiki_store as store import wiki_sync from conftest import WORKSPACE_SOURCE DOC = """# Zálohování Záloha běží každou noc přes rsync na druhý disk. """ TRAVEL_DOC = """# Tokijské metro Z Narity do centra jede Skyliner za osmatřicet minut. """ GIT_SOURCE = """ notes: kind: git url: {url} paths: ["**"] include: ["*.md"] exclude: - "**/node_modules/**" - "**/vendor/**" """ def _git(args, cwd=None): result = subprocess.run( ["git", "-c", "user.email=t@t", "-c", "user.name=t", *args], cwd=cwd, capture_output=True, text=True, check=True, ) return result.stdout class Remote: """A bare repo plus a working clone to push commits from.""" def __init__(self, root: Path): self.bare = root / "origin.git" self.work = root / "origin-work" _git(["init", "--bare", "--initial-branch=master", str(self.bare)]) _git(["clone", str(self.bare), str(self.work)]) @property def url(self) -> str: return str(self.bare) def commit(self, files: dict[str, str | None], message: str = "change") -> str: for rel_path, text in files.items(): path = self.work / rel_path if text is None: _git(["rm", "-q", rel_path], cwd=self.work) continue path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") _git(["add", rel_path], cwd=self.work) _git(["commit", "-m", message], cwd=self.work) _git(["push", "-q", "origin", "master"], cwd=self.work) return _git(["rev-parse", "HEAD"], cwd=self.work).strip() def move(self, old: str, new: str) -> str: (self.work / new).parent.mkdir(parents=True, exist_ok=True) _git(["mv", old, new], cwd=self.work) _git(["commit", "-m", "rename"], cwd=self.work) _git(["push", "-q", "origin", "master"], cwd=self.work) return _git(["rev-parse", "HEAD"], cwd=self.work).strip() def _use_git_source(env, url, with_workspace=False): sources = GIT_SOURCE.format(url=url) if with_workspace: sources += WORKSPACE_SOURCE env.write_config(sources) def _stats(env): with store.connection(env.db_path) as conn: return store.index_stats(conn) def test_first_run_clones_and_indexes(wiki_env, fake_embedder, tmp_path): remote = Remote(tmp_path) head = remote.commit({"zalohy.md": DOC, "japan/metro.md": TRAVEL_DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 clone = wiki_env.wiki_dir / "remote" / "notes" assert (clone / "zalohy.md").is_file() # non-bare: a working tree grep can read assert (clone / ".git").is_dir() with store.connection(wiki_env.db_path) as conn: assert sorted(store.list_source_files(conn, "notes")) == ["japan/metro.md", "zalohy.md"] source = store.get_source(conn, "notes") assert source is not None assert source["indexed_rev"] == head assert source["kind"] == "git" stats = _stats(wiki_env) assert stats["vectors"] == stats["chunks"] > 0 def test_second_run_over_the_same_revision_changes_nothing(wiki_env, fake_embedder, tmp_path): """Verification 2 for the git driver.""" remote = Remote(tmp_path) remote.commit({"zalohy.md": DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 before = _stats(wiki_env) log_before = wiki_env.log_text() assert wiki_sync.main([]) == 0 assert _stats(wiki_env) == before assert wiki_env.log_text() == log_before # ls-remote matched, so not even a fetch def test_new_commit_reindexes_only_the_changed_file(wiki_env, fake_embedder, tmp_path): remote = Remote(tmp_path) remote.commit({"zalohy.md": DOC, "japan/metro.md": TRAVEL_DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: metro_indexed_at = store.list_source_files(conn, "notes")["japan/metro.md"]["indexed_at"] head = remote.commit({"zalohy.md": DOC + "\n## Offsite\n\nKopie jede do S3.\n"}) assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: files = store.list_source_files(conn, "notes") assert files["japan/metro.md"]["indexed_at"] == metro_indexed_at # untouched source = store.get_source(conn, "notes") assert source is not None and source["indexed_rev"] == head assert len(store.bm25_ranked_ids(conn, "offsite", 10)) == 1 assert store.orphan_vector_ids(conn) == [] def test_deleted_file_upstream_drops_its_chunks_and_vectors(wiki_env, fake_embedder, tmp_path): remote = Remote(tmp_path) remote.commit({"zalohy.md": DOC, "japan/metro.md": TRAVEL_DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 remote.commit({"japan/metro.md": None}, message="drop metro") assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: assert sorted(store.list_source_files(conn, "notes")) == ["zalohy.md"] assert store.bm25_ranked_ids(conn, "skyliner", 10) == [] assert store.orphan_vector_ids(conn) == [] stats = _stats(wiki_env) assert stats["vectors"] == stats["chunks"] def test_renamed_file_moves_its_chunks(wiki_env, fake_embedder, tmp_path): remote = Remote(tmp_path) remote.commit({"japan/metro.md": TRAVEL_DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 remote.move("japan/metro.md", "japan/tokyo-metro.md") assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: assert sorted(store.list_source_files(conn, "notes")) == ["japan/tokyo-metro.md"] assert len(store.bm25_ranked_ids(conn, "skyliner", 10)) == 1 assert store.orphan_vector_ids(conn) == [] def test_commit_touching_nothing_indexed_only_records_the_rev(wiki_env, fake_embedder, tmp_path): remote = Remote(tmp_path) remote.commit({"zalohy.md": DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 before = _stats(wiki_env) head = remote.commit({"tool.py": "print('hello')\n"}) assert wiki_sync.main([]) == 0 assert _stats(wiki_env) == before with store.connection(wiki_env.db_path) as conn: source = store.get_source(conn, "notes") assert source is not None and source["indexed_rev"] == head assert "no indexed file changed" in wiki_env.log_text() def test_vendored_markdown_is_excluded(wiki_env, fake_embedder, tmp_path): remote = Remote(tmp_path) remote.commit( { "zalohy.md": DOC, "node_modules/pkg/README.md": "# Foreign readme\n\nnenasazovat\n", "web/vendor/lib/CHANGELOG.md": "# Changelog\n\nnenasazovat\n", } ) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: assert sorted(store.list_source_files(conn, "notes")) == ["zalohy.md"] def test_unreachable_remote_warns_skips_and_lets_other_sources_finish(wiki_env, fake_embedder, tmp_path): """Verification 8: per-source failure, indexed_rev untouched, workspace completes.""" _use_git_source(wiki_env, str(tmp_path / "does-not-exist.git"), with_workspace=True) wiki_env.write_file("notes/local.md", DOC) assert wiki_sync.main([]) == 1 log = wiki_env.log_text() assert "WARN notes:" in log # A permanent failure warns every tick, so each warning must stay a single line. assert len([line for line in log.splitlines() if "WARN notes:" in line]) == 1 assert all(line.strip() for line in log.splitlines()) with store.connection(wiki_env.db_path) as conn: assert store.list_source_files(conn, "notes") == {} git_source = store.get_source(conn, "notes") assert git_source is not None and git_source["indexed_rev"] is None # The workspace source, which has nothing to do with the network, finished. assert sorted(store.list_source_files(conn, "workspace")) == ["notes/local.md"] def test_remote_that_recovers_is_picked_up_on_the_next_tick(wiki_env, fake_embedder, tmp_path): missing = tmp_path / "later.git" _use_git_source(wiki_env, str(missing)) assert wiki_sync.main([]) == 1 remote = Remote(tmp_path / "real") (tmp_path / "real").mkdir(exist_ok=True) remote_bare = remote.bare remote.commit({"zalohy.md": DOC}) _use_git_source(wiki_env, str(remote_bare)) assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: assert sorted(store.list_source_files(conn, "notes")) == ["zalohy.md"] def test_missing_indexed_rev_falls_back_to_a_full_reindex(wiki_env, fake_embedder, tmp_path): """A force-pushed or gc'd base revision must degrade to a full pass, not crash.""" remote = Remote(tmp_path) remote.commit({"zalohy.md": DOC}) _use_git_source(wiki_env, remote.url) assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn, store.tx(conn): store.mark_synced(conn, "notes", "0" * 40, "2026-01-01T00:00:00+00:00") remote.commit({"japan/metro.md": TRAVEL_DOC}) assert wiki_sync.main([]) == 0 with store.connection(wiki_env.db_path) as conn: assert sorted(store.list_source_files(conn, "notes")) == ["japan/metro.md", "zalohy.md"] assert store.orphan_vector_ids(conn) == []