From 9e3201a296e936caa460bcbd72e57436ad86a362 Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 23 Aug 2026 11:22:20 -0400 Subject: [PATCH 1/3] feat(client): snapshot backups, restore points, scoped bulk pull Two modes per linked folder. 'mirror' keeps today's behaviour for a plain folder that add turned into a repo. 'snapshot' is new and is for a folder that already had a git history: nothing is ever committed on the user's behalf, and instead each pass builds a commit object from the working tree via a scratch index + commit-tree and pushes it to refs/granthi-backup//. HEAD, the index and every file stay exactly as the user left them, so uncommitted, unmerged, half-finished work leaves the machine with a timestamp to restore from. Verified on the beta forge (Gitea 1.27.2) that a custom ref namespace is accepted, readable via ls-remote, and absent from the branch list. Also: retention (all for 24h, hourly for 7d, daily beyond; unparseable timestamps kept), snapshots/restore commands, restore never writing over the working tree, get --all bounded by what the forge grants, list , .gitignore seeding, an add size guard, and a persisted device_id. 141 tests (was 108). --- README.md | 193 +++++++++- client/granthi_sync_client.py | 701 ++++++++++++++++++++++++++++++++-- tests/test_client.py | 446 ++++++++++++++++++++- 3 files changed, 1275 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index ffbde8d..487fe8d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# granthi-sync v1 +# granthi-sync v1.2 The signup → download → link-folders → cloud product spine for the Granthi forge, tested against the BETA forge (granthi-beta.shre.ai). Python 3 stdlib + @@ -50,10 +50,17 @@ cd granthi-sync # 3b. ...or push a local folder up. It becomes a private repo. ./bin/granthi-sync add ~/work/notes +# 3c. ...or pull down everything this account is allowed to see. +./bin/granthi-sync get --all --into ~/granthi + # 4. Keep everything synced. Autocommits, ff-pulls, pushes; skips anything # that has diverged rather than merging or forcing. ./bin/granthi-sync watch # --once for a single pass -./bin/granthi-sync status # what is linked, last sync, divergence +./bin/granthi-sync status # what is linked, mode, last sync + +# 5. Go back to how a folder looked at some point in time. +./bin/granthi-sync snapshots ~/work/notes +./bin/granthi-sync restore ~/work/notes --at 20260823T142530Z ``` Run `watch` as a background daemon on macOS with @@ -67,6 +74,79 @@ no account, no token, no partial state. Just run `link` again. to merge; when a folder shows `DIVERGED` in `status`, resolve it in git or on the forge web UI. The client will never force or auto-merge your work. +## Two modes, because two very different folders ask for this + +A folder people sync is either *their documents* or *their git project*, and +the correct behaviour is opposite in each case. Each linked folder therefore +carries a `mode`. + +| | `mirror` | `snapshot` | +|---|---|---| +| chosen for | a plain folder `add` turned into a repo | a folder that was already a git repo, and anything `get` clones | +| commits on your behalf | yes, `sync: ` | **never** | +| where work lands | the branch | `refs/granthi-backup//` | +| a restore point is | every commit | every snapshot | + +`snapshot` mode is what "the work may not be committed, but it is still +backed up" means in git terms. Each pass loads a scratch index from HEAD, +stages the working tree into *that* index, writes a tree, and commits it with +`commit-tree`. HEAD, your index, your stash and every file on disk are +untouched — you can be mid-rebase with a dirty tree and the backup still +records exactly what is on the disk right now. The user's history stays the +user's. + +Why a custom ref namespace: verified on the beta forge (Gitea 1.27.2) that +`refs/granthi-backup/...` is accepted, is readable through `ls-remote`, and +does **not** appear in the branch list. Under `refs/heads` a machine taking a +backup every 30 seconds would bury the branches a person actually made. + +Snapshots are parented on HEAD and deliberately **not** chained to the +previous snapshot: chaining would keep every old snapshot reachable from the +newest, so pruning a ref would free nothing and retention would be +decorative. + +**Retention** (or 30-second backups become a disk leak nobody can navigate): +everything is kept for 24 h, then thinned to hourly for 7 days, then daily. +Pruning runs at most hourly, per device, and only over that device's own +refs. A ref whose timestamp this version cannot parse is **kept** — deleting +the unrecognised is how a backup system loses the one thing someone needed. + +**Restore never writes over the working tree.** `restore` materialises a +restore point into a *new* directory and refuses a non-empty destination. +Someone restoring a backup is already having a bad day; overwriting the files +they still have would make the recovery tool the second disaster. + +## Device identity + +`link` mints a uuid on first run and persists it in `~/.granthi-sync/config.json` +as `device_id`, and sends it to `/v1/link`. Hostnames are neither stable +(people rename laptops) nor unique (every new mac is "Mac mini"), so a +hostname cannot key a backup ref or a device registry — two machines would +overwrite each other's snapshots. The service-side device registry is the +next phase; the client leads so the id already exists when it lands. + +## What "add the computer to the network" means — and does not + +The onboarding shape is: download → login → **the device is federated to the +account** → the device can reach its repos. + +The middle step is a *device registration*, not a network membership. Those +sound like one step and must not be built as one: this estate's tailnet is a +single flat private network carrying the granthi VPS, aros-vps, the Shadow +box and the Mac. Putting a customer's laptop on it to let them sync a folder +would hand that laptop L3 reach to every piece of infrastructure we run. + +So: + +* **Our own machines** may join the tailnet — that is an operator action with + an operator's judgement behind it. +* **Customer devices never do.** Their transport is public HTTPS to + granthi-link and the forge through cloudflared. That is the same exposure + step already in the promotion window below, and it is what makes a genuinely + new computer able to onboard itself at all — today `link` only works from + inside the tailnet, which means "you can't set up a new computer without an + operator first" is the honest status. + ## Components ### `server/granthi_link.py` — provisioning service (granthi VPS) @@ -211,13 +291,29 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): (`authorization_pending`/`slow_down` handled), then calls `/v1/link`. `--token` skips the device flow with a ready Zitadel token (headless/dev). Result stored in `~/.granthi-sync/config.json` (0600). -* `list` — every repo the linked token can see, with the local folder each - is already synced to. Reads `GET /api/v1/user/repos` on the forge +* `list [pattern]` — every repo the linked token can see, with the local + folder each is already synced to. `pattern` narrows the table by name + (substring, or a glob like `work-*`), case-insensitively, against both + `owner/name` and the bare name. Filtering is display-only: the set already + came from the forge under this account's token. Reads + `GET /api/v1/user/repos` on the forge **directly** with the scoped user token — no granthi-link round-trip, so the read path needs no service change. Pagination is followed to a short page; if the `FORGE_MAX_PAGES` guard trips, the output says the list is incomplete rather than letting a bounded page read as the whole set. -* `get [--into DIR]` — the download half of `add`. Clones +* `get --all [--into DIR] [--mode M]` — clone every repo this account can + see, skipping the ones already linked here. **"Only the repos they are + granted" needs no client-side permission logic**: `/api/v1/user/repos` is + evaluated by the forge against this account's own scoped token, so the + list *is* the grant. A client-side filter would be a second opinion about + someone else's authorisation. One repo failing does not abandon the rest, + and a truncated listing is reported loudly — `--all` must never quietly + mean "the first 2000". +* `get [--into DIR] [--mode M]` — the download half of + `add`. Defaults to `snapshot` mode unless the repo carries a + `.granthi-sync.json` marker saying otherwise, so a plain synced folder + behaves the same on the second machine while someone's real project is + never autocommitted onto. Clones with `--origin granthi` (the remote name `watch` looks for) and `-c credential.helper=…` (the repo does not exist yet, so the helper cannot be installed first; git also persists it into the new config), then @@ -225,22 +321,55 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): so a cloned repo is picked up by `watch` immediately. Refuses a non-empty destination. Branch is read with `symbolic-ref` (an empty repo has an unborn HEAD) and falls back to `main`. -* `add [--name N] [--private|--public]` — `git init -b main` if - needed, creates the cloud repo via `/v1/repos`, adds remote `granthi`, - initial commit + push. The token is delivered by a **git credential - helper** (the client's hidden `git-credential` subcommand reading the 0600 - config) — never embedded in the remote URL (estate rule). -* `watch [--interval 30] [--once]` — per folder: autocommit - (`sync: `) → fetch → ff-pull if remote strictly ahead → push if - local strictly ahead. **DIVERGED → log + record + SKIP. Never force, never - merge** — the same policy as the mesh. SIGTERM-clean. -* `status` — table of linked folders, last sync, divergence flags. +* `add [--name N] [--private|--public] [--mode M] [--force]` — + `git init -b main` if needed, creates the cloud repo via `/v1/repos`, adds + remote `granthi`, initial commit + push. The token is delivered by a **git + credential helper** (the client's hidden `git-credential` subcommand + reading the 0600 config) — never embedded in the remote URL (estate rule). + Pushes the folder's **current** branch, not a hardcoded `main`: an existing + repo may sit on `master` or a feature branch, and publishing that work + under the wrong name is not a cosmetic error. + Two guards, because `add -A` takes whatever it is given: a starter + `.gitignore` is seeded when the folder has none (an existing one is never + touched — it is the user's), and a folder over 20 000 files / 512 MB is + refused unless `--force`. The seeded ignore file covers `.env`, `*.key`, + `*.pem`, `id_rsa` and friends, and it governs snapshots too — the scratch + index honours `.gitignore` exactly as a normal commit does. +* `watch [--interval 30] [--once]` — per folder, by mode. `mirror`: + autocommit (`sync: `) → fetch → ff-pull if remote strictly ahead → + push if local strictly ahead. `snapshot`: fetch → push a snapshot of the + working tree to this device's backup ref → ff-pull only when the tree is + clean (local edits are already safe on the backup ref, so it reports and + leaves the tree alone rather than failing) → push the user's own commits + when they are strictly ahead. **DIVERGED → log + record + SKIP. Never + force, never merge** — the same policy as the mesh — **but the backup + still happens**, because divergence is when work is most at risk. + Retention pruning runs at most hourly. SIGTERM-clean. +* `snapshots [--limit 20]` — restore points, newest first. Read from + the **remote**, not a local cache: the feature exists for the case where + this machine is gone. +* `restore --at [--into DIR]` — materialise one restore + point into a new directory; refuses a non-empty destination. +* `status` — table of linked folders, mode, last sync, divergence flags. * Run as a daemon on macOS with `client/launchd/ai.granthi.sync.plist` (edit the script path, then `launchctl bootstrap gui/$UID `). ## Tests -* `python3 -m unittest discover -s tests` — 65 tests: autocommit/ff/diverged +* `python3 -m unittest discover -s tests` — 141 tests. The v1.2 additions + cover: a snapshot capturing uncommitted work while HEAD, the index and the + working tree stay byte-identical; snapshots landing outside `refs/heads`; + an unchanged tree not being re-pushed; a diverged folder still being backed + up; a dirty tree blocking the ff-pull but not the backup; retention keeping + everything recent, thinning to hourly then daily, and **keeping** + unparseable timestamps; prune deleting only the thinned refs; `.gitignore` + seeding never overwriting an existing one and keeping `.env` out of + snapshots; the folder-size guard being bounded rather than walking the + disk; mode detection; `list` filtering; `get --all` skipping what is + already present, defaulting to snapshot mode, and shouting about + truncation; `restore` writing a new folder, refusing a non-empty + destination, and leaving the working tree alone. +* Earlier suite: autocommit/ff/diverged logic against real temp git repos (including "diverged never touches the remote"), config 0600 handling (including umask-proof creation and a no-chmod guard), credential-helper quoting/injection, mocked device-flow @@ -267,6 +396,38 @@ the provisioning service creates their forge account + scoped token on the fly — the forge never sees a password and the user never sees the forge admin. Every linked folder becomes a private repo under their account. +## Next phase — invites and per-repo access (designed, not built) + +Today `/v1/link` creates an account and every folder becomes a private repo +under it. What is missing is the multi-person case: an existing account +inviting somebody, and that person's device waking up with access to *some* +repos and not others. + +Shape this should take, so the next session does not re-litigate it: + +* **Where grants live: granthi-link's own store, not Zitadel orgs.** The + estate's house pattern is app-side tenancy tables with the IdP only + providing identity (see the AROS `tenants` / `tenant_members` split). + Grants therefore sit beside `state.json`, and **Gitea is the enforcement + point** — a grant is materialised as a repo collaborator or an org team + membership, so the forge itself refuses unauthorised reads. Nothing in the + client decides access, which is why `get --all` needs no permission logic. +* **Token scope does not change.** `write:repository,write:user` stays; per + repo permission is collaborator/team state, not a token property. +* **`POST /v1/invite`** (account admin → new member): creates the shre-id + user (Zitadel admin PAT, on aros-vps at + `/opt/shre-id/deploy/secrets/shre_id_zitadel_pat`), records the intended + grants, and returns an invite the person redeems by running `link`. Until + they redeem it, nothing exists on the forge. +* **`POST /v1/grants`** (account admin): add/remove repo access for a member; + applies the change to Gitea and records it. Removing a grant must also + remove the collaborator — a grant store that drifts from the forge is + worse than no store. +* Both endpoints are account-admin-only and rate-limited like `/v1/link`. +* The grant store inherits the same fragility already noted for the rate + limiter: a flat JSON file behind an in-process lock, fine for one + `ThreadingHTTPServer` and **not** fine the day this runs multi-process. + ## Promotion window (beta → prod) 1. **Expose :3042** behind cloudflared (granthi.shre.ai vhost or diff --git a/client/granthi_sync_client.py b/client/granthi_sync_client.py index 7ecaac0..5c101a7 100644 --- a/client/granthi_sync_client.py +++ b/client/granthi_sync_client.py @@ -7,32 +7,62 @@ Commands: the headless path), then POST /v1/link on the granthi-link service. Stores {server, gitea_base, login, token, token_name} in ~/.granthi-sync/config.json (0600). - list - Table of every repo the linked token can see on the forge, with the - local folder each one is already synced to (if any). - get [--into DIR] + list [pattern] + Table of every repo the linked token can see on the forge -- which + is exactly the set the forge grants this account, never a + client-side filter over someone else's repos -- with the local + folder each one is already synced to (if any). `pattern` narrows + the table by name (substring, or a glob like 'work-*'). + get [--into DIR] [--mode M] | --all [--into DIR] Clone a forge repo and register it for `watch` -- the download half of `add`. Remote is named 'granthi' at clone time and the token is supplied by the credential helper, never embedded in the URL. - add [--name N] [--private/--public] + `--all` pulls every repo this account can see, skipping the ones + already linked on this machine. + add [--name N] [--private/--public] [--mode M] [--force] git init (branch main) if needed, create the cloud repo via /v1/repos, add remote 'granthi', initial commit + push. The token is supplied via a git credential helper (this script's hidden `git-credential` subcommand), never embedded in the remote URL. + Seeds a starter .gitignore and refuses an unexpectedly large folder + unless --force. watch [--interval 30] [--once] - Poll linked folders: autocommit local changes, fetch, ff-pull when - the remote is strictly ahead, push when local is strictly ahead. - DIVERGED branches are logged + recorded and SKIPPED -- never force, - never merge (same policy as the estate's gitea_sync.py mesh). - SIGTERM-clean. + Poll linked folders. Per folder, by MODE (below): `mirror` + autocommits local changes and pushes them; `snapshot` never commits + for you and instead mirrors the working tree to a per-device backup + ref. Both ff-pull when the remote is strictly ahead. DIVERGED + branches are logged + recorded and SKIPPED -- never force, never + merge (same policy as the estate's gitea_sync.py mesh) -- but a + diverged folder is still backed up. SIGTERM-clean. + snapshots + List the restore points held for a folder, newest first. + restore --at [--into DIR] + Materialise one restore point into a NEW directory. Never writes + over the working tree. status - Table of linked folders, last sync, divergence flags. + Table of linked folders, mode, last sync, divergence flags. + +MODES + mirror The folder IS the repo. Local changes are autocommitted as + `sync: ` and pushed; every commit is a restore point. + Chosen automatically for a plain folder that `add` turned into + a repo -- the "my documents live in the cloud" case. + snapshot The folder has a real git history that belongs to the user. + NOTHING is committed on their behalf. Each pass builds a commit + object from the working tree without touching HEAD, the index + or any file, and pushes it to + refs/granthi-backup//. Uncommitted, unmerged, + half-finished work is therefore off the machine with a + timestamp to restore from, while the user's own history stays + exactly as they left it. Chosen automatically for a folder that + was already a git repo, and for anything `get` clones. Stdlib + git CLI only. """ import argparse import base64 +import fnmatch import json import os import re @@ -40,13 +70,15 @@ import shlex import signal import subprocess import sys +import tempfile import time import urllib.error import urllib.parse import urllib.request -from datetime import datetime, timezone +import uuid +from datetime import datetime, timedelta, timezone -VERSION = "1.1.0" +VERSION = "1.2.0" CONFIG_DIR = os.path.expanduser(os.environ.get("GRANTHI_SYNC_HOME", "~/.granthi-sync")) CONFIG_PATH = os.path.join(CONFIG_DIR, "config.json") @@ -61,6 +93,50 @@ DEVICE_SCOPE = "openid profile email" FORGE_PAGE_LIMIT = 50 FORGE_MAX_PAGES = 40 # 2000 repos; a guard against an unbounded paging loop +# Backup refs live OUTSIDE refs/heads on purpose: verified on the beta forge +# (Gitea 1.27.2) that a custom namespace is accepted, is readable through +# ls-remote, and does NOT appear in the branch list or the branch dropdown. +# Under refs/heads a machine taking a snapshot every 30s would bury the +# user's real branches. +BACKUP_NS = "refs/granthi-backup" +SNAPSHOT_TS_FMT = "%Y%m%dT%H%M%SZ" + +# Retention. Unbounded snapshots are a disk leak with no way to use them, so +# thin them by age: everything recent, then hourly, then daily. +KEEP_ALL_HOURS = 24 +KEEP_HOURLY_DAYS = 7 +PRUNE_EVERY_SECONDS = 3600 + +# `add` guard rails. A plain `git add -A` on the wrong folder commits every +# byte in it, so refuse the surprising cases unless the user insists. +ADD_MAX_BYTES = 512 * 1024 * 1024 +ADD_MAX_FILES = 20000 +MARKER_FILE = ".granthi-sync.json" + +# Seeded only when the folder has no .gitignore of its own. The point is the +# secret-shaped entries: the rest is noise reduction. +DEFAULT_GITIGNORE = """\ +# Seeded by granthi-sync. Edit freely -- it is yours now. +.DS_Store +Thumbs.db +*.swp +node_modules/ +__pycache__/ +.venv/ +venv/ + +# Never sync credentials. +.env +.env.* +*.key +*.pem +*.p12 +id_rsa +id_ed25519 +credentials.json +secrets.json +""" + def log(msg): ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -109,6 +185,21 @@ def load_config(): return {} +def device_id(cfg): + """Stable per-installation id, minted once and persisted. + + The hostname is neither stable (people rename laptops) nor unique (every + fresh mac is 'Mac mini'), so it cannot key a backup ref or a device + registry -- two machines would write each other's snapshots. Callers get + the id from here; `save_config` is theirs to call. + """ + existing = cfg.get("device_id") + if existing: + return existing + cfg["device_id"] = uuid.uuid4().hex + return cfg["device_id"] + + def save_config(cfg): os.makedirs(CONFIG_DIR, mode=0o700, exist_ok=True) tmp = CONFIG_PATH + ".tmp" @@ -125,10 +216,14 @@ def save_config(cfg): # git helpers # -------------------------------------------------------------------------- -def git(folder, *args, check=True): +def git(folder, *args, check=True, env=None): """Run git in folder. Returns (rc, stdout). Never uses --force.""" + run_env = None + if env: + run_env = dict(os.environ) + run_env.update(env) proc = subprocess.run(["git", "-C", folder] + list(args), - capture_output=True, text=True) + capture_output=True, text=True, env=run_env) if check and proc.returncode != 0: raise RuntimeError( f"git {' '.join(args)} failed in {folder}: {proc.stderr.strip()}") @@ -156,6 +251,181 @@ def autocommit(folder): return True +# -------------------------------------------------------------------------- +# snapshots -- back up work that was never committed +# -------------------------------------------------------------------------- + +def _head_sha(folder): + rc, sha = git(folder, "rev-parse", "--verify", "--quiet", "HEAD", + check=False) + return sha.strip() if rc == 0 and sha.strip() else None + + +def build_snapshot(folder): + """Commit the working tree WITHOUT touching HEAD, the index or a file. + + A scratch index is loaded from HEAD, `add -A` stages the working tree + into *that* index, and commit-tree writes a commit object parented on + HEAD. The user's own index and history are untouched -- they can be + mid-rebase with a dirty tree and this still records what is on disk. + + Returns (commit_sha, tree_sha) or None when the tree is empty. + """ + head = _head_sha(folder) + git_dir = git(folder, "rev-parse", "--absolute-git-dir")[1].strip() + fd, tmp_index = tempfile.mkstemp(prefix="granthi-index-", dir=git_dir) + os.close(fd) + os.unlink(tmp_index) # git wants to create it itself + env = {"GIT_INDEX_FILE": tmp_index} + try: + if head: + git(folder, "read-tree", head, env=env) + # Honours .gitignore, so a seeded ignore file keeps secrets out of + # snapshots exactly as it keeps them out of commits. + git(folder, "add", "-A", env=env) + _, tree = git(folder, "write-tree", env=env) + tree = tree.strip() + if not tree: + return None + if head: + _, head_tree = git(folder, "rev-parse", f"{head}^{{tree}}") + if head_tree.strip() == tree: + # Nothing uncommitted: HEAD already holds this exact content. + return None + ts = datetime.now(timezone.utc).isoformat(timespec="seconds") + args = ["commit-tree", tree, "-m", f"granthi snapshot: {ts}"] + if head: + args += ["-p", head] + # Snapshots are parented on HEAD and nothing else -- deliberately NOT + # chained to the previous snapshot. Chaining would keep every old + # snapshot reachable from the newest one, so pruning a ref would free + # nothing and retention would be decorative. + _, commit = git(folder, *args, + env={"GIT_AUTHOR_NAME": "granthi-sync", + "GIT_AUTHOR_EMAIL": "sync@granthi.local", + "GIT_COMMITTER_NAME": "granthi-sync", + "GIT_COMMITTER_EMAIL": "sync@granthi.local"}) + return commit.strip(), tree + finally: + if os.path.exists(tmp_index): + os.unlink(tmp_index) + + +def snapshot_ref(dev, ts=None): + ts = ts or datetime.now(timezone.utc).strftime(SNAPSHOT_TS_FMT) + return f"{BACKUP_NS}/{dev}/{ts}", ts + + +def push_snapshot(folder, dev, remote="granthi"): + """Take a snapshot and push it to this device's backup namespace. + + Returns the ref pushed, or None when there was nothing new to back up. + """ + built = build_snapshot(folder) + if not built: + return None + commit, tree = built + if tree == _last_snapshot_tree(folder, dev): + return None # working tree unchanged since the last backup + ref, _ = snapshot_ref(dev) + git(folder, "push", remote, f"{commit}:{ref}") + _remember_snapshot_tree(folder, dev, tree) + return ref + + +def _tree_marker_path(folder, dev): + git_dir = git(folder, "rev-parse", "--absolute-git-dir")[1].strip() + return os.path.join(git_dir, f"granthi-last-snapshot-{dev}") + + +def _last_snapshot_tree(folder, dev): + try: + with open(_tree_marker_path(folder, dev)) as f: + return f.read().strip() + except OSError: + return None + + +def _remember_snapshot_tree(folder, dev, tree): + try: + with open(_tree_marker_path(folder, dev), "w") as f: + f.write(tree) + except OSError: + pass # a lost marker costs one redundant push, never correctness + + +def list_snapshots(folder, dev, remote="granthi"): + """Backup refs for this device on the remote, newest first. + + Read from the REMOTE, not a local cache: the point of the feature is + surviving the loss of this machine, so what the server holds is the + answer that matters. + """ + rc, out = git(folder, "ls-remote", remote, f"{BACKUP_NS}/{dev}/*", + check=False) + if rc != 0: + return [] + snaps = [] + for line in out.splitlines(): + sha, _, ref = line.partition("\t") + ref = ref.strip() + if not ref.startswith(f"{BACKUP_NS}/{dev}/"): + continue + snaps.append({"sha": sha.strip(), "ref": ref, + "ts": ref.rsplit("/", 1)[-1]}) + return sorted(snaps, key=lambda s: s["ts"], reverse=True) + + +def _parse_ts(ts): + try: + return datetime.strptime(ts, SNAPSHOT_TS_FMT).replace(tzinfo=timezone.utc) + except ValueError: + return None + + +def snapshots_to_prune(snaps, now=None): + """Retention: keep everything for KEEP_ALL_HOURS, then one per hour for + KEEP_HOURLY_DAYS, then one per day. Returns the refs to delete. + + Unparseable timestamps are KEPT. A ref this version does not understand + is not evidence it is worthless, and deleting the unrecognised is how a + backup system loses the one thing someone needed. + """ + now = now or datetime.now(timezone.utc) + keep, drop, seen = [], [], set() + for snap in sorted(snaps, key=lambda s: s["ts"], reverse=True): + when = _parse_ts(snap["ts"]) + if when is None: + keep.append(snap) + continue + age = now - when + if age <= timedelta(hours=KEEP_ALL_HOURS): + keep.append(snap) + continue + bucket = (when.strftime("%Y%m%dT%H") + if age <= timedelta(days=KEEP_HOURLY_DAYS) + else when.strftime("%Y%m%d")) + if bucket in seen: + drop.append(snap) + else: + seen.add(bucket) + keep.append(snap) + return [s["ref"] for s in drop] + + +def prune_snapshots(folder, dev, remote="granthi", now=None): + """Delete thinned-out backup refs on the remote. Returns how many went.""" + stale = snapshots_to_prune(list_snapshots(folder, dev, remote), now=now) + if not stale: + return 0 + # Deleting a ref is not force-pushing over anyone's work: these refs are + # written by this device alone and hold no history but its own. + for batch_start in range(0, len(stale), 50): + batch = stale[batch_start:batch_start + 50] + git(folder, "push", remote, *[f":{ref}" for ref in batch], check=False) + return len(stale) + + def sync_state(folder, remote="granthi", branch="main"): """Classify local vs remote after a fetch. @@ -182,30 +452,60 @@ def sync_state(folder, remote="granthi", branch="main"): return "diverged" -def sync_folder(folder, remote="granthi", branch="main"): +def worktree_dirty(folder): + _, status = git(folder, "status", "--porcelain") + return bool(status) + + +def sync_folder(folder, remote="granthi", branch="main", mode="mirror", + dev=None): """One sync pass for one folder. Returns (outcome, detail). outcome in {'synced', 'pushed', 'clean', 'diverged', 'error'}. Policy: ff-only pulls, plain pushes. NEVER --force, NEVER merge. + + In `snapshot` mode nothing is committed on the user's behalf; their + uncommitted work is pushed to a backup ref instead, and that happens on + EVERY outcome -- including diverged and including a tree too dirty to + ff-pull. Those are exactly the moments work is most at risk, so they are + the last moments to skip a backup. """ try: - committed = autocommit(folder) + backed_up = None + committed = False + if mode == "mirror": + committed = autocommit(folder) git(folder, "fetch", remote) + if mode == "snapshot" and dev: + backed_up = push_snapshot(folder, dev, remote) + + def note(base): + if backed_up: + return f"{base}; backed up {backed_up.rsplit('/', 1)[-1]}" + return base + state = sync_state(folder, remote, branch) ff_pulled = False if state == "remote-ahead-ff": + if mode == "snapshot" and worktree_dirty(folder): + # A ff-pull would fail (or clobber) with local edits present. + # The edits are already safe on the backup ref, so report and + # leave the working tree exactly as the user left it. + return "clean", note("remote ahead; local edits present, " + "not pulling") git(folder, "merge", "--ff-only", f"refs/remotes/{remote}/{branch}") ff_pulled = True state = sync_state(folder, remote, branch) if state == "diverged": - return "diverged", "local and remote both advanced; skipping (no force, no merge)" + return "diverged", note("local and remote both advanced; " + "skipping (no force, no merge)") if state in ("local-ahead", "no-remote-branch"): git(folder, "push", "-u", remote, branch) - return "pushed", "committed+pushed" if committed else "pushed" + return "pushed", note("committed+pushed" if committed else "pushed") # state == "in-sync" if ff_pulled: - return "synced", "ff-pulled" - return "clean", "in-sync" + return "synced", note("ff-pulled") + return "clean", note("in-sync") except RuntimeError as e: return "error", str(e) @@ -283,13 +583,19 @@ def device_flow(): def cmd_link(args): zitadel_token = args.token or device_flow() + cfg = load_config() + dev = device_id(cfg) status, resp = http_json( "POST", f"{args.server.rstrip('/')}/v1/link", body={"zitadel_access_token": zitadel_token, - "device_name": args.device}) + "device_name": args.device, + # Sent so the service can keep a device registry (and so a + # device can later be listed or revoked by something other + # than a hostname). A service that does not know the field + # ignores it, so the client can lead. + "device_id": dev}) if status != 200: raise SystemExit(f"link failed (HTTP {status}): {resp}") - cfg = load_config() cfg.update({"server": args.server.rstrip("/"), "gitea_base": resp["gitea_base"], "login": resp["login"], @@ -298,7 +604,8 @@ def cmd_link(args): cfg.setdefault("folders", {}) save_config(cfg) log(f"linked as {resp['login']} on {resp['gitea_base']} " - f"(token {resp['token_name']}); config: {CONFIG_PATH}") + f"(token {resp['token_name']}, device {dev[:8]}); " + f"config: {CONFIG_PATH}") return 0 @@ -346,9 +653,35 @@ def list_repos(cfg): return repos, bool(fetch(FORGE_MAX_PAGES + 1)) +def match_repo(repo, pattern): + """Case-insensitive: glob if the pattern has glob syntax, else substring. + + Matched against full_name AND the bare name, so 'notes' finds + 'alice/notes' without anyone having to think about owners. + """ + if not pattern: + return True + full = (repo.get("full_name") or "").lower() + name = (repo.get("name") or full.rsplit("/", 1)[-1]).lower() + pat = pattern.lower() + if any(c in pat for c in "*?["): + return fnmatch.fnmatch(full, pat) or fnmatch.fnmatch(name, pat) + return pat in full or pat in name + + def cmd_list(args): cfg = require_linked(load_config()) repos, truncated = list_repos(cfg) + pattern = getattr(args, "pattern", None) + if pattern: + # Filter for display only. The set came from the forge under this + # account's own token, so it is already the permitted set -- this + # narrows what is shown, it does not widen what is reachable. + repos = [r for r in repos if match_repo(r, pattern)] + if not repos: + print(f"no repos matching {pattern!r}" + + (" (and the listing was truncated)" if truncated else "")) + return 0 if not repos: print("no repos on the forge for this account") return 0 @@ -396,11 +729,25 @@ def parse_repo_arg(repo, login): return "/".join(parts) -def cmd_get(args): - cfg = require_linked(load_config()) - full_name = parse_repo_arg(args.repo, cfg["login"]) +def read_marker(folder): + """Intent `add` left in the repo, if any: {'mode': ...}. + + Carrying the mode in the repo is what makes a plain synced folder behave + the same on the second machine. Without it, `get` would have to guess, + and guessing 'mirror' on someone's real project is the destructive + direction. + """ + try: + with open(os.path.join(folder, MARKER_FILE)) as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + +def clone_one(cfg, full_name, dest, mode=None): + """Clone one repo and register it. Returns the folder meta written.""" name = full_name.rsplit("/", 1)[-1] - dest = os.path.abspath(args.into or name) if os.path.exists(dest) and os.listdir(dest): raise SystemExit(f"refusing to clone into a non-empty path: {dest}") clone_url = f"{cfg['gitea_base'].rstrip('/')}/{full_name}.git" @@ -420,23 +767,151 @@ def cmd_get(args): rc, branch = git(dest, "symbolic-ref", "--short", "HEAD", check=False) if rc != 0 or not branch: branch = "main" - cfg.setdefault("folders", {})[dest] = { + # Default to `snapshot`: a cloned repo carries history that is the + # user's, and autocommitting onto it is the one mistake that cannot be + # undone quietly. `add` marks the folders that genuinely want mirroring. + chosen = mode or read_marker(dest).get("mode") or "snapshot" + meta = { "name": name, "full_name": full_name, "branch": branch, + "mode": chosen, "last_sync": datetime.now(timezone.utc).isoformat(timespec="seconds"), "diverged": False} + cfg.setdefault("folders", {})[dest] = meta save_config(cfg) - log(f"cloned {full_name} -> {dest} (branch {branch}); " + log(f"cloned {full_name} -> {dest} (branch {branch}, mode {chosen}); " f"`granthi-sync watch` will keep it synced") + return meta + + +def cmd_get(args): + cfg = require_linked(load_config()) + device_id(cfg) + if getattr(args, "all", False): + return _get_all(cfg, args) + if not args.repo: + raise SystemExit("give a repo name, or --all") + full_name = parse_repo_arg(args.repo, cfg["login"]) + name = full_name.rsplit("/", 1)[-1] + clone_one(cfg, full_name, os.path.abspath(args.into or name), args.mode) return 0 +def _get_all(cfg, args): + """Pull every repo this account can see. + + 'Only the repos they are granted' needs no client-side permission logic: + /api/v1/user/repos is evaluated by the forge against this account's own + scoped token, so the list IS the grant. Anything else would be a second + opinion about someone else's authorisation. + """ + repos, truncated = list_repos(cfg) + base = os.path.abspath(args.into or ".") + linked = set(cfg.get("folders", {})) + cloned = skipped = failed = 0 + for repo in sorted(repos, key=lambda r: r.get("full_name") or ""): + full_name = repo.get("full_name") + if not full_name: + continue + dest = os.path.join(base, full_name.rsplit("/", 1)[-1]) + if dest in linked or (os.path.exists(dest) and os.listdir(dest)): + log(f"skip {full_name}: already present at {dest}") + skipped += 1 + continue + try: + clone_one(cfg, full_name, dest, args.mode) + cloned += 1 + except SystemExit as e: # one bad repo must not abandon the rest + log(f"FAILED {full_name}: {e}") + failed += 1 + log(f"get --all: {cloned} cloned, {skipped} already here, {failed} failed") + if truncated: + log(f"WARNING: the forge listing stopped after {FORGE_MAX_PAGES} " + f"pages of {FORGE_PAGE_LIMIT} -- this was NOT every repo you " + f"have access to.") + return 1 if failed else 0 + + +def measure_folder(folder, max_files=ADD_MAX_FILES): + """(files, bytes) below .git, stopping once max_files is exceeded. + + Bounded on purpose: the caller only needs to know whether the folder is + surprisingly big, and walking a 2 TB drive to answer that would be the + same mistake as syncing it. + """ + files = total = 0 + for root, dirs, names in os.walk(folder): + dirs[:] = [d for d in dirs if d != ".git"] + for n in names: + path = os.path.join(root, n) + try: + if os.path.islink(path): + continue + total += os.path.getsize(path) + except OSError: + continue + files += 1 + if files > max_files: + return files, total + return files, total + + +def seed_gitignore(folder): + """Write a starter .gitignore when the folder has none. Returns bool. + + `add -A` takes everything it is not told to leave, and the folders + people sync are exactly the ones holding a stray .env or id_rsa. An + existing .gitignore is never touched -- it is the user's. + """ + path = os.path.join(folder, ".gitignore") + if os.path.exists(path): + return False + with open(path, "w") as f: + f.write(DEFAULT_GITIGNORE) + return True + + +def detect_mode(folder, had_git): + """`mirror` for a plain folder we turned into a repo; `snapshot` for one + that already had a history worth protecting.""" + if not had_git: + return "mirror" + return "snapshot" if _head_sha(folder) else "mirror" + + +def write_marker(folder, mode): + with open(os.path.join(folder, MARKER_FILE), "w") as f: + json.dump({"mode": mode, "created_by": f"granthi-sync/{VERSION}"}, + f, indent=2) + f.write("\n") + + def cmd_add(args): cfg = require_linked(load_config()) folder = os.path.abspath(args.folder) if not os.path.isdir(folder): raise SystemExit(f"no such folder: {folder}") name = args.name or re.sub(r"[^a-zA-Z0-9._-]+", "-", os.path.basename(folder)) + had_git = os.path.isdir(os.path.join(folder, ".git")) + if not args.force: + files, size = measure_folder(folder) + if files > ADD_MAX_FILES or size > ADD_MAX_BYTES: + raise SystemExit( + f"{folder} holds {files} files / {size // (1024 * 1024)} MB, " + f"over the {ADD_MAX_FILES} file / " + f"{ADD_MAX_BYTES // (1024 * 1024)} MB guard.\n" + f"Add a .gitignore for what should not sync, point at a " + f"narrower folder, or re-run with --force if this is really " + f"what you want.") ensure_repo(folder) + mode = args.mode or detect_mode(folder, had_git) + if seed_gitignore(folder): + log(f"seeded a starter .gitignore in {folder} -- review it; " + f"anything listed there will NOT be synced") + if mode == "mirror": + # Only mirror folders carry the marker: it tells the next machine's + # `get` that autocommitting here is wanted. A snapshot folder is + # someone's real project and gets no file dropped into it. + write_marker(folder, mode) install_credential_helper(folder) status, resp = http_json( "POST", f"{cfg['server']}/v1/repos", @@ -453,14 +928,30 @@ def cmd_add(args): git(folder, "remote", "set-url", "granthi", clone_url) else: git(folder, "remote", "add", "granthi", clone_url) - autocommit(folder) - git(folder, "push", "-u", "granthi", "main") + # An existing repo may sit on master, or on a feature branch. Pushing + # its work to a hardcoded 'main' would publish the wrong thing under the + # right name. + rc, branch = git(folder, "symbolic-ref", "--short", "HEAD", check=False) + branch = branch.strip() if rc == 0 and branch.strip() else "main" + if mode == "mirror": + autocommit(folder) + git(folder, "push", "-u", "granthi", branch) + dev = device_id(cfg) + if mode == "snapshot": + ref = push_snapshot(folder, dev) + if ref: + log(f"backed up uncommitted work to {ref}") cfg.setdefault("folders", {})[folder] = { - "name": name, "full_name": f"{cfg['login']}/{name}", "branch": "main", + "name": name, "full_name": f"{cfg['login']}/{name}", "branch": branch, + "mode": mode, "last_sync": datetime.now(timezone.utc).isoformat(timespec="seconds"), "diverged": False} save_config(cfg) - log(f"linked folder {folder} -> {clone_url}") + log(f"linked folder {folder} -> {clone_url} (mode {mode}, branch {branch})") + if mode == "snapshot": + log("mode snapshot: your commits stay yours -- granthi-sync will " + "never commit for you here, and backs up uncommitted work to " + f"{BACKUP_NS}/{dev[:8]}…/") return 0 @@ -473,13 +964,23 @@ def _sigterm(signum, frame): log(f"signal {signum} received; finishing current pass then exiting") -def watch_pass(): +_LAST_PRUNE = 0.0 + + +def watch_pass(now=None): + global _LAST_PRUNE cfg = load_config() + dev = device_id(cfg) + # Pruning talks to the remote for every folder, so it runs on a slow + # clock of its own rather than on every 30s pass. + due = (now or time.time()) - _LAST_PRUNE >= PRUNE_EVERY_SECONDS for folder, meta in sorted(cfg.get("folders", {}).items()): if not os.path.isdir(folder): log(f"SKIP {folder}: missing") continue - outcome, detail = sync_folder(folder, branch=meta.get("branch", "main")) + mode = meta.get("mode", "mirror") + outcome, detail = sync_folder(folder, branch=meta.get("branch", "main"), + mode=mode, dev=dev) meta["diverged"] = outcome == "diverged" if outcome in ("pushed", "synced"): meta["last_sync"] = datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -489,6 +990,16 @@ def watch_pass(): log(f"ERROR {folder}: {detail}") else: log(f"{outcome} {folder}: {detail}") + if due and mode == "snapshot" and outcome != "error": + try: + gone = prune_snapshots(folder, dev) + if gone: + log(f"pruned {gone} old restore points in {folder}") + except RuntimeError as e: + # Retention failing must never stop the backup itself. + log(f"prune failed in {folder} (backups unaffected): {e}") + if due: + _LAST_PRUNE = now or time.time() save_config(cfg) @@ -509,6 +1020,88 @@ def cmd_watch(args): return 0 +def _folder_meta(cfg, folder): + path = os.path.abspath(folder) + meta = cfg.get("folders", {}).get(path) + if not meta: + raise SystemExit(f"{path} is not a linked folder " + f"(see: granthi-sync status)") + return path, meta + + +def cmd_snapshots(args): + """Restore points, newest first. Both modes answer the same question.""" + cfg = require_linked(load_config()) + folder, meta = _folder_meta(cfg, args.folder) + dev = device_id(cfg) + if meta.get("mode", "mirror") == "mirror": + rc, out = git(folder, "log", "--format=%H\t%cI\t%s", "-n", + str(args.limit), check=False) + if rc != 0 or not out: + print("no restore points yet") + return 0 + print(f"restore points for {folder} (mode mirror: every commit is one)") + for line in out.splitlines(): + sha, ts, subject = (line.split("\t") + ["", ""])[:3] + print(f" {ts} {sha[:12]} {subject}") + return 0 + snaps = list_snapshots(folder, dev) + if not snaps: + print("no restore points yet (nothing uncommitted has been backed up)") + return 0 + print(f"restore points for {folder} (mode snapshot, device {dev[:8]}):") + for snap in snaps[:args.limit]: + print(f" {snap['ts']} {snap['sha'][:12]}") + if len(snaps) > args.limit: + print(f" ... {len(snaps) - args.limit} older (use --limit)") + return 0 + + +def cmd_restore(args): + """Materialise a restore point into a NEW directory. + + Never into the working tree. Someone restoring a backup is already + having a bad day; overwriting the files they still have would be how a + recovery tool becomes the second disaster. + """ + cfg = require_linked(load_config()) + folder, meta = _folder_meta(cfg, args.folder) + dev = device_id(cfg) + target = args.at + sha = None + if meta.get("mode", "mirror") == "snapshot": + for snap in list_snapshots(folder, dev): + if snap["ts"] == target or snap["sha"].startswith(target): + sha = snap["sha"] + git(folder, "fetch", "granthi", f"{snap['ref']}") + break + if sha is None: + rc, resolved = git(folder, "rev-parse", "--verify", "--quiet", + f"{target}^{{commit}}", check=False) + if rc != 0 or not resolved.strip(): + raise SystemExit( + f"no restore point {target!r} for {folder} " + f"(see: granthi-sync snapshots {folder})") + sha = resolved.strip() + dest = os.path.abspath( + args.into or f"{folder.rstrip(os.sep)}-restore-{target}") + if os.path.exists(dest) and os.listdir(dest): + raise SystemExit(f"refusing to restore over a non-empty path: {dest}") + os.makedirs(dest, exist_ok=True) + proc = subprocess.run(["git", "-C", folder, "archive", "--format=tar", sha], + capture_output=True) + if proc.returncode != 0: + raise SystemExit( + f"could not read {sha[:12]}: {proc.stderr.decode(errors='replace').strip()}") + untar = subprocess.run(["tar", "-x", "-C", dest], input=proc.stdout, + capture_output=True) + if untar.returncode != 0: + raise SystemExit( + f"could not write {dest}: {untar.stderr.decode(errors='replace').strip()}") + log(f"restored {sha[:12]} -> {dest} (your working tree was not touched)") + return 0 + + def cmd_status(args): cfg = load_config() if "login" in cfg: @@ -520,12 +1113,13 @@ def cmd_status(args): if not folders: print("no folders linked") return 0 - rows = [("FOLDER", "REPO", "LAST SYNC", "DIVERGED")] + rows = [("FOLDER", "REPO", "MODE", "LAST SYNC", "DIVERGED")] for folder, meta in sorted(folders.items()): rows.append((folder, meta.get("name", "?"), + meta.get("mode", "mirror"), meta.get("last_sync", "never"), "YES" if meta.get("diverged") else "no")) - widths = [max(len(r[i]) for r in rows) for i in range(4)] + widths = [max(len(r[i]) for r in rows) for i in range(len(rows[0]))] for r in rows: print(" ".join(c.ljust(w) for c, w in zip(r, widths))) return 0 @@ -544,21 +1138,48 @@ def main(argv=None): sp.set_defaults(fn=cmd_link) sp = sub.add_parser("list", help="list forge repos this account can see") + sp.add_argument("pattern", nargs="?", + help="filter by name: substring, or a glob like 'work-*'") sp.set_defaults(fn=cmd_list) sp = sub.add_parser("get", help="clone a forge repo and keep it synced") - sp.add_argument("repo", help="repo name, or owner/repo") - sp.add_argument("--into", help="target folder (default: ./)") + sp.add_argument("repo", nargs="?", help="repo name, or owner/repo") + sp.add_argument("--all", action="store_true", + help="clone every repo this account can see") + sp.add_argument("--into", + help="target folder (default: ./; with --all, the " + "directory to clone into)") + sp.add_argument("--mode", choices=("mirror", "snapshot"), + help="override the sync mode (default: snapshot, or " + "whatever the repo was marked with)") sp.set_defaults(fn=cmd_get) sp = sub.add_parser("add", help="link a folder and push it to the cloud") sp.add_argument("folder") sp.add_argument("--name") + sp.add_argument("--mode", choices=("mirror", "snapshot"), + help="override the sync mode (default: mirror for a plain " + "folder, snapshot for an existing git repo)") + sp.add_argument("--force", action="store_true", + help="skip the folder-size guard") grp = sp.add_mutually_exclusive_group() grp.add_argument("--private", dest="private", action="store_true", default=True) grp.add_argument("--public", dest="private", action="store_false") sp.set_defaults(fn=cmd_add) + sp = sub.add_parser("snapshots", help="list restore points for a folder") + sp.add_argument("folder") + sp.add_argument("--limit", type=int, default=20) + sp.set_defaults(fn=cmd_snapshots) + + sp = sub.add_parser("restore", + help="materialise a restore point into a NEW folder") + sp.add_argument("folder") + sp.add_argument("--at", required=True, + help="timestamp or sha from `granthi-sync snapshots`") + sp.add_argument("--into", help="destination (default: -restore-)") + sp.set_defaults(fn=cmd_restore) + sp = sub.add_parser("watch", help="sync loop over linked folders") sp.add_argument("--interval", type=int, default=30) sp.add_argument("--once", action="store_true", help="single pass then exit") diff --git a/tests/test_client.py b/tests/test_client.py index aa3159d..7d19213 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -33,6 +33,17 @@ def run_git(cwd, *args): capture_output=True, text=True, env=GIT_ENV).stdout.strip() + +def get_ns(**kw): + """Namespace for cmd_get with the parser's defaults filled in, so a test + exercises the same shape argparse hands the command.""" + kw.setdefault("all", False) + kw.setdefault("mode", None) + kw.setdefault("into", None) + kw.setdefault("repo", None) + return argparse.Namespace(**kw) + + class GitScenarioBase(unittest.TestCase): """bare 'cloud' repo + two working clones to simulate device vs remote.""" @@ -341,7 +352,7 @@ class TestGet(GitScenarioBase): def test_get_clones_registers_and_is_watchable(self): dest = os.path.join(self.tmp, "pulled") - client.cmd_get(argparse.Namespace(repo="cloud", into=dest)) + client.cmd_get(get_ns(repo="cloud", into=dest)) # cloned content self.assertTrue(os.path.exists(os.path.join(dest, "hello.txt"))) @@ -366,7 +377,7 @@ class TestGet(GitScenarioBase): def test_get_accepts_owner_qualified_name(self): dest = os.path.join(self.tmp, "pulled2") - client.cmd_get(argparse.Namespace(repo="alice/cloud", into=dest)) + client.cmd_get(get_ns(repo="alice/cloud", into=dest)) self.assertTrue(os.path.exists(os.path.join(dest, "hello.txt"))) def test_get_refuses_non_empty_destination(self): @@ -374,14 +385,14 @@ class TestGet(GitScenarioBase): os.makedirs(dest) self.write(dest, "mine.txt", "do not clobber") with self.assertRaises(SystemExit): - client.cmd_get(argparse.Namespace(repo="cloud", into=dest)) + client.cmd_get(get_ns(repo="cloud", into=dest)) self.assertEqual(open(os.path.join(dest, "mine.txt")).read(), "do not clobber") def test_get_unlinked_exits_like_add(self): with mock.patch.object(client, "load_config", lambda: {}): with self.assertRaises(SystemExit) as cm: - client.cmd_get(argparse.Namespace(repo="cloud", into=None)) + client.cmd_get(get_ns(repo="cloud", into=None)) self.assertIn("not linked", str(cm.exception)) def test_get_empty_repo_falls_back_to_main(self): @@ -389,7 +400,7 @@ class TestGet(GitScenarioBase): subprocess.run(["git", "init", "--bare", "-b", "main", empty], check=True, capture_output=True, env=GIT_ENV) dest = os.path.join(self.tmp, "blank") - client.cmd_get(argparse.Namespace(repo="blank", into=dest)) + client.cmd_get(get_ns(repo="blank", into=dest)) meta = client.load_config()["folders"][os.path.abspath(dest)] self.assertEqual(meta["branch"], "main") @@ -399,17 +410,17 @@ class TestGet(GitScenarioBase): "", "alice/"]: with self.subTest(repo=bad): with self.assertRaises(SystemExit): - client.cmd_get(argparse.Namespace(repo=bad, into=None)) + client.cmd_get(get_ns(repo=bad, into=None)) def test_get_records_full_name_so_list_matches_the_right_owner(self): dest = os.path.join(self.tmp, "pulled4") - client.cmd_get(argparse.Namespace(repo="cloud", into=dest)) + client.cmd_get(get_ns(repo="cloud", into=dest)) meta = client.load_config()["folders"][os.path.abspath(dest)] self.assertEqual(meta["full_name"], "alice/cloud") def test_list_does_not_mark_a_same_named_other_owner_repo_as_local(self): dest = os.path.join(self.tmp, "pulled5") - client.cmd_get(argparse.Namespace(repo="cloud", into=dest)) + client.cmd_get(get_ns(repo="cloud", into=dest)) cfg = client.load_config() repos = [{"name": "cloud", "full_name": "alice/cloud", "private": True, "updated_at": "2026-08-20T00:00:00Z"}, @@ -426,9 +437,426 @@ class TestGet(GitScenarioBase): def test_get_never_puts_token_in_remote_url(self): dest = os.path.join(self.tmp, "pulled3") - client.cmd_get(argparse.Namespace(repo="cloud", into=dest)) + client.cmd_get(get_ns(repo="cloud", into=dest)) self.assertNotIn("sekrit", run_git(dest, "remote", "get-url", "granthi")) +class TestSnapshots(GitScenarioBase): + """The whole point of snapshot mode: work that was never committed still + leaves the machine, and the user's own history is not touched.""" + + DEV = "dev0123456789" + + def test_snapshot_captures_uncommitted_work_without_moving_head(self): + self.write(self.local, "a.txt", "committed") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "real commit") + head_before = run_git(self.local, "rev-parse", "HEAD") + self.write(self.local, "a.txt", "UNCOMMITTED EDIT") + self.write(self.local, "new.txt", "never staged") + status_before = run_git(self.local, "status", "--porcelain") + # (run_git strips, so the leading space of ' M' is gone here) + self.assertEqual(status_before, "M a.txt\n?? new.txt") + + commit, tree = client.build_snapshot(self.local) + + self.assertEqual(run_git(self.local, "rev-parse", "HEAD"), head_before) + # the index is untouched: a.txt is still merely modified, not staged, + # and new.txt is still untracked. A snapshot that quietly staged the + # user's files would corrupt whatever they were in the middle of. + self.assertEqual(run_git(self.local, "status", "--porcelain"), + status_before) + # working tree still holds exactly what the user left there + with open(os.path.join(self.local, "a.txt")) as f: + self.assertEqual(f.read(), "UNCOMMITTED EDIT") + # ...and the snapshot commit holds it too + blob = run_git(self.local, "show", f"{commit}:a.txt") + self.assertEqual(blob, "UNCOMMITTED EDIT") + self.assertIn("new.txt", run_git(self.local, "ls-tree", "--name-only", + tree)) + self.assertEqual(run_git(self.local, "log", "-1", "--format=%P", + commit), head_before) + + def test_snapshot_is_none_when_nothing_is_uncommitted(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + self.assertIsNone(client.build_snapshot(self.local)) + + def test_push_snapshot_lands_in_the_backup_namespace(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + self.write(self.local, "a.txt", "work in progress") + + ref = client.push_snapshot(self.local, self.DEV) + + self.assertTrue(ref.startswith(f"refs/granthi-backup/{self.DEV}/"), ref) + refs = run_git(self.bare, "for-each-ref", "--format=%(refname)") + self.assertIn(ref, refs.splitlines()) + # it is NOT a branch: the user's branch list stays theirs + self.assertNotIn("refs/heads/granthi-backup", refs) + + def test_push_snapshot_skips_an_unchanged_tree(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + self.write(self.local, "a.txt", "work in progress") + self.assertIsNotNone(client.push_snapshot(self.local, self.DEV)) + self.assertIsNone(client.push_snapshot(self.local, self.DEV)) + + def test_snapshot_mode_never_commits_for_the_user(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "mine") + run_git(self.local, "push", "-u", "granthi", "main") + head_before = run_git(self.local, "rev-parse", "HEAD") + self.write(self.local, "a.txt", "dirty") + + outcome, detail = client.sync_folder(self.local, mode="snapshot", + dev=self.DEV) + + self.assertEqual(run_git(self.local, "rev-parse", "HEAD"), head_before) + self.assertIn("backed up", detail) + self.assertEqual(outcome, "clean") + self.assertTrue(run_git(self.local, "status", "--porcelain")) + + def test_diverged_folder_is_still_backed_up(self): + """Divergence is when work is most at risk -- the least acceptable + moment to skip the backup.""" + self.write(self.local, "a.txt", "one") + client.sync_folder(self.local) # mirror push to establish the branch + other = self.other_clone() + self.write(other, "b.txt", "remote side") + run_git(other, "add", "-A") + run_git(other, "commit", "-m", "remote") + run_git(other, "push", "origin", "main") + remote_sha = run_git(other, "rev-parse", "HEAD") + self.write(self.local, "a.txt", "local side") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "local") + self.write(self.local, "c.txt", "and uncommitted too") + + outcome, detail = client.sync_folder(self.local, mode="snapshot", + dev=self.DEV) + + self.assertEqual(outcome, "diverged") + self.assertIn("backed up", detail) + self.assertEqual(run_git(self.bare, "rev-parse", "main"), remote_sha) + snaps = client.list_snapshots(self.local, self.DEV) + self.assertEqual(len(snaps), 1) + self.assertIn("c.txt", run_git(self.local, "ls-tree", "--name-only", + snaps[0]["sha"])) + + def test_dirty_tree_blocks_the_pull_but_not_the_backup(self): + self.write(self.local, "a.txt", "one") + client.sync_folder(self.local) + other = self.other_clone() + self.write(other, "b.txt", "remote side") + run_git(other, "add", "-A") + run_git(other, "commit", "-m", "remote") + run_git(other, "push", "origin", "main") + self.write(self.local, "wip.txt", "half-finished") + head_before = run_git(self.local, "rev-parse", "HEAD") + + outcome, detail = client.sync_folder(self.local, mode="snapshot", + dev=self.DEV) + + self.assertEqual(run_git(self.local, "rev-parse", "HEAD"), head_before) + self.assertFalse(os.path.exists(os.path.join(self.local, "b.txt"))) + self.assertIn("not pulling", detail) + self.assertIn("backed up", detail) + + def test_snapshots_are_listed_from_the_remote(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + self.write(self.local, "a.txt", "wip") + ref = client.push_snapshot(self.local, self.DEV) + snaps = client.list_snapshots(self.local, self.DEV) + self.assertEqual([s["ref"] for s in snaps], [ref]) + self.assertEqual(client.list_snapshots(self.local, "someone-else"), []) + + +class TestRetention(unittest.TestCase): + """Retention is what keeps 30-second backups from being a disk leak -- + and what must never quietly eat the one restore point someone needs.""" + + NOW = client.datetime(2026, 8, 23, 12, 0, 0, tzinfo=client.timezone.utc) + + def snap(self, when): + ts = when.strftime(client.SNAPSHOT_TS_FMT) + return {"ts": ts, "ref": f"refs/granthi-backup/d/{ts}", "sha": "x"} + + def test_everything_recent_is_kept(self): + snaps = [self.snap(self.NOW - client.timedelta(minutes=m)) + for m in range(0, 24 * 60, 30)] + self.assertEqual(client.snapshots_to_prune(snaps, now=self.NOW), []) + + def test_older_than_a_day_thins_to_hourly(self): + base = self.NOW - client.timedelta(days=2) + snaps = [self.snap(base + client.timedelta(minutes=m)) + for m in (0, 10, 20, 60, 70)] + pruned = client.snapshots_to_prune(snaps, now=self.NOW) + self.assertEqual(len(pruned), 3) # 5 in 2 hourly buckets -> keep 2 + + def test_older_than_a_week_thins_to_daily(self): + base = self.NOW - client.timedelta(days=30) + snaps = [self.snap(base + client.timedelta(hours=h)) + for h in (0, 1, 2, 25)] + pruned = client.snapshots_to_prune(snaps, now=self.NOW) + self.assertEqual(len(pruned), 2) # 2 days -> keep 1 each + + def test_unparseable_timestamps_are_kept_not_deleted(self): + snaps = [{"ts": "not-a-timestamp", + "ref": "refs/granthi-backup/d/not-a-timestamp", "sha": "x"}] + self.assertEqual(client.snapshots_to_prune(snaps, now=self.NOW), []) + + +class TestPrune(GitScenarioBase): + DEV = "devprune" + + def test_prune_deletes_stale_refs_on_the_remote(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + sha = run_git(self.local, "rev-parse", "HEAD") + old = "20260101T000000Z" + older = "20260101T001000Z" + for ts in (old, older): + run_git(self.local, "push", "granthi", + f"{sha}:refs/granthi-backup/{self.DEV}/{ts}") + self.assertEqual(len(client.list_snapshots(self.local, self.DEV)), 2) + + gone = client.prune_snapshots(self.local, self.DEV) + + self.assertEqual(gone, 1) # same hour, older one dropped + left = client.list_snapshots(self.local, self.DEV) + self.assertEqual([s["ts"] for s in left], [older]) + + +class TestAddGuards(GitScenarioBase): + def test_gitignore_is_seeded_only_when_absent(self): + self.assertTrue(client.seed_gitignore(self.local)) + with open(os.path.join(self.local, ".gitignore")) as f: + body = f.read() + self.assertIn(".env", body) + with open(os.path.join(self.local, ".gitignore"), "w") as f: + f.write("mine-only\n") + self.assertFalse(client.seed_gitignore(self.local)) + with open(os.path.join(self.local, ".gitignore")) as f: + self.assertEqual(f.read(), "mine-only\n") + + def test_seeded_gitignore_keeps_secrets_out_of_snapshots(self): + client.seed_gitignore(self.local) + self.write(self.local, ".env", "SECRET=hunter2") + self.write(self.local, "ok.txt", "fine") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + self.write(self.local, "ok.txt", "changed") + commit, tree = client.build_snapshot(self.local) + names = run_git(self.local, "ls-tree", "-r", "--name-only", tree) + self.assertIn("ok.txt", names) + self.assertNotIn(".env", names.splitlines()) + + def test_measure_folder_stops_counting_past_the_cap(self): + for i in range(12): + self.write(self.local, f"f{i}.txt", "x" * 10) + files, size = client.measure_folder(self.local, max_files=5) + self.assertEqual(files, 6) # bounded: stopped one past the cap + self.assertLess(size, 12 * 10) + + def test_measure_folder_ignores_dot_git(self): + files, _ = client.measure_folder(self.local) + self.assertEqual(files, 0) + + def test_detect_mode(self): + plain = os.path.join(self.tmp, "plain") + os.makedirs(plain) + self.assertEqual(client.detect_mode(plain, had_git=False), "mirror") + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "real history") + self.assertEqual(client.detect_mode(self.local, had_git=True), + "snapshot") + empty = os.path.join(self.tmp, "empty-repo") + os.makedirs(empty) + client.ensure_repo(empty) + self.assertEqual(client.detect_mode(empty, had_git=True), "mirror") + + +class TestMatchRepo(unittest.TestCase): + def repo(self, full): + return {"full_name": full, "name": full.split("/")[-1]} + + def test_substring_is_case_insensitive_and_matches_bare_name(self): + self.assertTrue(client.match_repo(self.repo("alice/Notes"), "notes")) + self.assertTrue(client.match_repo(self.repo("alice/notes"), "ALICE")) + self.assertFalse(client.match_repo(self.repo("alice/notes"), "ledger")) + + def test_glob_syntax_switches_to_glob(self): + self.assertTrue(client.match_repo(self.repo("alice/work-2026"), + "work-*")) + self.assertFalse(client.match_repo(self.repo("alice/homework"), + "work-*")) + + def test_empty_pattern_matches_everything(self): + self.assertTrue(client.match_repo(self.repo("alice/x"), None)) + + +class TestRestore(GitScenarioBase): + DEV = "devrestore" + + def setUp(self): + super().setUp() + client.save_config({"gitea_base": self.tmp, "login": "alice", + "token": "sekrit", "device_id": self.DEV, + "folders": {self.local: { + "name": "cloud", "full_name": "alice/cloud", + "branch": "main", "mode": "snapshot"}}}) + + def test_restore_writes_a_new_folder_and_leaves_the_working_tree_alone(self): + self.write(self.local, "a.txt", "original") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + self.write(self.local, "a.txt", "the version I want back") + ref = client.push_snapshot(self.local, self.DEV) + ts = ref.rsplit("/", 1)[-1] + self.write(self.local, "a.txt", "what I have now") + dest = os.path.join(self.tmp, "restored") + + client.cmd_restore(argparse.Namespace(folder=self.local, at=ts, + into=dest)) + + with open(os.path.join(dest, "a.txt")) as f: + self.assertEqual(f.read(), "the version I want back") + with open(os.path.join(self.local, "a.txt")) as f: + self.assertEqual(f.read(), "what I have now") + + def test_restore_refuses_a_non_empty_destination(self): + self.write(self.local, "a.txt", "one") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + self.write(self.local, "a.txt", "two") + ref = client.push_snapshot(self.local, self.DEV) + busy = os.path.join(self.tmp, "busy") + os.makedirs(busy) + with open(os.path.join(busy, "keepme"), "w") as f: + f.write("do not clobber") + with self.assertRaises(SystemExit): + client.cmd_restore(argparse.Namespace( + folder=self.local, at=ref.rsplit("/", 1)[-1], into=busy)) + self.assertTrue(os.path.exists(os.path.join(busy, "keepme"))) + + def test_unknown_restore_point_is_an_error_not_an_empty_folder(self): + with self.assertRaises(SystemExit): + client.cmd_restore(argparse.Namespace( + folder=self.local, at="20990101T000000Z", into=None)) + + def test_restore_refuses_a_folder_that_is_not_linked(self): + with self.assertRaises(SystemExit): + client.cmd_restore(argparse.Namespace( + folder=os.path.join(self.tmp, "nowhere"), at="x", into=None)) + + +class TestDeviceId(unittest.TestCase): + def test_device_id_is_stable_and_persisted(self): + cfg = {} + first = client.device_id(cfg) + self.assertEqual(client.device_id(cfg), first) + self.assertEqual(cfg["device_id"], first) + + def test_two_installs_get_different_ids(self): + self.assertNotEqual(client.device_id({}), client.device_id({})) + + +class TestGetAll(GitScenarioBase): + """--all must pull exactly what the forge grants, and one bad repo must + not abandon the rest.""" + + def setUp(self): + super().setUp() + self.forge = os.path.join(self.tmp, "forge") + for full in ("alice/one", "alice/two"): + path = os.path.join(self.forge, full + ".git") + os.makedirs(os.path.dirname(path), exist_ok=True) + subprocess.run(["git", "init", "--bare", "-b", "main", path], + check=True, capture_output=True, env=GIT_ENV) + seed = os.path.join(self.tmp, "seed-" + full.replace("/", "-")) + subprocess.run(["git", "clone", path, seed], check=True, + capture_output=True, env=GIT_ENV) + run_git(seed, "config", "user.name", "s") + run_git(seed, "config", "user.email", "s@s") + self.write(seed, "f.txt", full) + run_git(seed, "add", "-A") + run_git(seed, "commit", "-m", "seed") + run_git(seed, "push", "origin", "main") + client.save_config({"gitea_base": self.forge, "login": "alice", + "token": "sekrit", "folders": {}}) + self.repos = [{"name": "one", "full_name": "alice/one"}, + {"name": "two", "full_name": "alice/two"}] + + def test_all_clones_every_granted_repo(self): + into = os.path.join(self.tmp, "workspace") + os.makedirs(into) + with mock.patch.object(client, "list_repos", + lambda c: (self.repos, False)): + rc = client.cmd_get(get_ns(all=True, into=into)) + self.assertEqual(rc, 0) + for name in ("one", "two"): + self.assertTrue(os.path.exists(os.path.join(into, name, "f.txt"))) + self.assertEqual(len(client.load_config()["folders"]), 2) + + def test_all_skips_what_is_already_here(self): + into = os.path.join(self.tmp, "workspace2") + os.makedirs(into) + with mock.patch.object(client, "list_repos", + lambda c: (self.repos, False)): + client.cmd_get(get_ns(all=True, into=into)) + with mock.patch("sys.stdout", new_callable=io.StringIO) as out: + client.cmd_get(get_ns(all=True, into=into)) + self.assertIn("already present", out.getvalue()) + self.assertEqual(len(client.load_config()["folders"]), 2) + + def test_all_defaults_cloned_repos_to_snapshot_mode(self): + into = os.path.join(self.tmp, "workspace3") + os.makedirs(into) + with mock.patch.object(client, "list_repos", + lambda c: (self.repos, False)): + client.cmd_get(get_ns(all=True, into=into)) + modes = {m["mode"] for m in client.load_config()["folders"].values()} + self.assertEqual(modes, {"snapshot"}) + + def test_all_says_so_loudly_when_the_listing_was_truncated(self): + into = os.path.join(self.tmp, "workspace4") + os.makedirs(into) + with mock.patch.object(client, "list_repos", + lambda c: (self.repos, True)), \ + mock.patch("sys.stdout", new_callable=io.StringIO) as out: + client.cmd_get(get_ns(all=True, into=into)) + self.assertIn("NOT every repo", out.getvalue()) + + +class TestMarkerRoundTrip(GitScenarioBase): + """A plain folder synced on machine A must behave the same on machine B: + the intent travels in the repo, not in one machine's config.""" + + def test_marker_written_by_add_makes_get_choose_mirror(self): + client.write_marker(self.local, "mirror") + self.assertEqual(client.read_marker(self.local)["mode"], "mirror") + + def test_missing_or_corrupt_marker_falls_back_to_the_safe_mode(self): + self.assertEqual(client.read_marker(self.local), {}) + with open(os.path.join(self.local, client.MARKER_FILE), "w") as f: + f.write("{not json") + self.assertEqual(client.read_marker(self.local), {}) + + if __name__ == "__main__": unittest.main() From ddb829d701b3d3bf5627d24401e79aeae4f9027e Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 23 Aug 2026 11:29:30 -0400 Subject: [PATCH 2/3] fix(client): make our credential helper the only one the repo consults Live QA against the beta forge failed its first push with 'Failed to authenticate user' while the config held a valid token. Cause: credential.helper is a list accumulated across system/global/repo config, and this machine has osxkeychain (Xcode gitconfig) plus store (~/.gitconfig). A stale entry for the forge host answered before our helper. The same list is a token leak in the other direction: git calls approve on every helper after a successful auth, so 'store' writes the forge token into ~/.git-credentials in plaintext -- undoing the 0600 config and the no-token-in-URL rule. Confirmed accidentally during QA when a verification clone with a URL-embedded token re-created exactly that entry. Fix: set an empty credential.helper first (git reads that as 'forget the inherited list'), then add ours -- in install_credential_helper and in the git clone inside get. 2 regression tests, one of which drives 'git credential fill' against a poisoned outer helper. 143 tests. --- README.md | 31 ++++++++++++++++++++++++-- client/granthi_sync_client.py | 27 +++++++++++++++++++++- tests/test_client.py | 42 +++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 487fe8d..3c58f44 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,31 @@ restore point into a *new* directory and refuses a non-empty destination. Someone restoring a backup is already having a bad day; overwriting the files they still have would make the recovery tool the second disaster. +## The credential helper must be the ONLY helper (found by live QA) + +`credential.helper` is a list that accumulates across system, global and repo +config, and git asks every helper in it. A stock mac already has two — +`osxkeychain` from Xcode's gitconfig, and `store` from many people's +`~/.gitconfig` — and they lose in both directions: + +* **reading:** a stale entry for the forge host answers before our helper, so + pushes fail `remote: Failed to authenticate user` long after the token was + rotated, and nothing in this tool's config explains why. This is exactly + how the first live-QA run failed; +* **writing:** git calls `approve` on every helper after a successful auth, + so `store` copies the forge token into `~/.git-credentials` **in + plaintext**. Keeping the token in a 0600 file and out of remote URLs buys + nothing if git then hands it to a plaintext store. + +So `install_credential_helper` (and the `git clone` in `get`) sets an **empty** +`credential.helper` first, which resets the inherited list, then adds ours. +Exactly one helper serves this repo. + +Corollary worth remembering: a token embedded in a remote URL gets saved by +`store` on first use. During QA a verification clone with a URL-embedded +token re-created the very entry that had just been cleaned out. That is the +whole reason this client passes tokens through a helper and never a URL. + ## Device identity `link` mints a uuid on first run and persists it in `~/.granthi-sync/config.json` @@ -356,7 +381,7 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): ## Tests -* `python3 -m unittest discover -s tests` — 141 tests. The v1.2 additions +* `python3 -m unittest discover -s tests` — 143 tests. The v1.2 additions cover: a snapshot capturing uncommitted work while HEAD, the index and the working tree stay byte-identical; snapshots landing outside `refs/heads`; an unchanged tree not being re-pushed; a diverged folder still being backed @@ -368,7 +393,9 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): disk; mode detection; `list` filtering; `get --all` skipping what is already present, defaulting to snapshot mode, and shouting about truncation; `restore` writing a new folder, refusing a non-empty - destination, and leaving the working tree alone. + destination, and leaving the working tree alone; and the credential helper + being the only one the repo consults, proven by driving + `git credential fill` against a deliberately poisoned outer helper. * Earlier suite: autocommit/ff/diverged logic against real temp git repos (including "diverged never touches the remote"), config 0600 handling (including umask-proof creation and a diff --git a/client/granthi_sync_client.py b/client/granthi_sync_client.py index 5c101a7..953ee57 100644 --- a/client/granthi_sync_client.py +++ b/client/granthi_sync_client.py @@ -544,7 +544,27 @@ def credential_helper_value(): def install_credential_helper(folder): - git(folder, "config", "credential.helper", credential_helper_value()) + """Make OUR helper the only one this repo consults. + + credential.helper is a LIST that accumulates across system, global and + repo config, and git asks every helper in order. On a stock mac there are + already two (`osxkeychain` from Xcode's gitconfig, `store` from many + people's ~/.gitconfig), and they lose both ways: + + * reading -- a stale entry for the forge host answers first, so pushes + fail with "Failed to authenticate user" long after the token was + rotated, and nothing in this tool's config explains why; + * writing -- git calls `approve` on every helper after a successful + auth, so `store` copies the forge token into ~/.git-credentials in + PLAINTEXT. Keeping the token in a 0600 file and out of remote URLs + is pointless if git hands it to a plaintext store on first use. + + An empty value resets the inherited list, so replace-all-then-add leaves + exactly one helper: this script. + """ + git(folder, "config", "--replace-all", "credential.helper", "") + git(folder, "config", "--add", "credential.helper", + credential_helper_value()) # -------------------------------------------------------------------------- @@ -756,7 +776,12 @@ def clone_one(cfg, full_name, dest, mode=None): # persists it into the new repo's config. --origin names the remote # 'granthi' up front so `watch` picks the folder up without a rename. proc = subprocess.run( + # The empty -c resets the inherited helper list (see + # install_credential_helper) so a stale keychain/store entry cannot + # answer for the forge host during the clone, and the token cannot + # leak into a plaintext store afterwards. ["git", "clone", + "-c", "credential.helper=", "-c", f"credential.helper={credential_helper_value()}", "--origin", "granthi", clone_url, dest], capture_output=True, text=True) diff --git a/tests/test_client.py b/tests/test_client.py index 7d19213..d360328 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -858,5 +858,47 @@ class TestMarkerRoundTrip(GitScenarioBase): self.assertEqual(client.read_marker(self.local), {}) +class TestCredentialHelperIsolation(GitScenarioBase): + """A repo-local helper is not enough on a normal machine: git consults + system + global helpers too, and they both shadow us and copy the token + into plaintext. Found by live QA against the beta forge, not by a unit + test -- so it gets one now.""" + + def test_install_leaves_exactly_one_helper(self): + run_git(self.local, "config", "--add", "credential.helper", "store") + client.install_credential_helper(self.local) + # --get-all merges system + global + local, so entries inherited from + # the machine still appear. What matters is that the last two are the + # reset and ours: git reads an empty value as "forget every helper + # inherited so far", so nothing before it can answer. + helpers = run_git(self.local, "config", "--get-all", + "credential.helper").splitlines() + self.assertEqual(helpers[-2], "", helpers) + self.assertIn("git-credential", helpers[-1]) + # the repo-level 'store' this test added is gone, not merely outvoted + self.assertNotIn("store", helpers) + + def test_inherited_helper_cannot_answer_for_the_forge(self): + """The end-to-end property: with a poisoned outer helper configured, + the credential git actually resolves is ours.""" + fake = os.path.join(self.tmp, "poison.sh") + with open(fake, "w") as f: + f.write("#!/bin/sh\n" + "echo username=wrong-user\necho password=stale-token\n") + os.chmod(fake, 0o755) + run_git(self.local, "config", "--add", "credential.helper", + f"!{shlex.quote(fake)}") + client.save_config({"gitea_base": "http://forge.example:3041", + "login": "alice", "token": "the-right-token"}) + client.install_credential_helper(self.local) + out = subprocess.run( + ["git", "-C", self.local, "credential", "fill"], + input="protocol=http\nhost=forge.example:3041\n\n", + capture_output=True, text=True, env=dict( + GIT_ENV, GRANTHI_SYNC_HOME=os.environ["GRANTHI_SYNC_HOME"])) + self.assertIn("password=the-right-token", out.stdout) + self.assertNotIn("stale-token", out.stdout) + + if __name__ == "__main__": unittest.main() From eb70ca08ad0efa0c3cb303bfd4ce229bf62ff1da Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 23 Aug 2026 11:42:19 -0400 Subject: [PATCH 3/3] fix(client): make the disaster-recovery path actually work Review found the read path scoped to the CURRENT device's uuid, which breaks the exact case snapshot mode exists for: when the laptop dies, the replacement machine has a new id, so snapshots printed 'no restore points yet' while the backups sat on the forge, and restore errored. Reproduced, then fixed by unscoping the READ only. Writing stays device-scoped (two machines must not overwrite each other) and pruning stays device-scoped (machine A must not apply its clock to machine B's refs); the docstring now says why the three differ. Also from the same review: - mirror mode printed a %cI timestamp that restore could not accept, so copying the first column looped the user back to snapshots. It now matches the log, and refuses an ambiguous timestamp (two commits in one second) with the candidate ids instead of guessing. - the size guard advised 'add a .gitignore' while measuring with a plain walk that ignored one. It now measures what git would sync, through a throwaway git dir outside the folder so a refused add leaves no .git behind. - get --all caught only SystemExit, so a RuntimeError from any git call abandoned the remaining repos. - get --all mapped alice/notes and bob/notes to one path and reported the second as 'already present'. Clashes now clone to - and say so. - the prune clock was in-memory, so watch --once under launchd pruned every run. Persisted in config. 153 tests. Live-verified on the beta forge: machine A backed up uncommitted work and was deleted; machine B, different device id, cloned the repo, listed A's snapshot and restored both files. --- README.md | 34 +++++- client/granthi_sync_client.py | 170 +++++++++++++++++++++++----- tests/test_client.py | 202 ++++++++++++++++++++++++++++++++++ 3 files changed, 374 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 3c58f44..deca7c5 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,10 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): list *is* the grant. A client-side filter would be a second opinion about someone else's authorisation. One repo failing does not abandon the rest, and a truncated listing is reported loudly — `--all` must never quietly - mean "the first 2000". + mean "the first 2000". `alice/notes` and `bob/notes` both want + `/notes`; the second is cloned to `/bob-notes` and the clash is + logged, because reporting it as "already present" would leave the user + believing they had pulled both. * `get [--into DIR] [--mode M]` — the download half of `add`. Defaults to `snapshot` mode unless the repo carries a `.granthi-sync.json` marker saying otherwise, so a plain synced folder @@ -360,6 +363,12 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): refused unless `--force`. The seeded ignore file covers `.env`, `*.key`, `*.pem`, `id_rsa` and friends, and it governs snapshots too — the scratch index honours `.gitignore` exactly as a normal commit does. + The size guard measures what git *would* sync, ignore rules included + (including the machine's global excludes), because its own advice is "add + a .gitignore for what should not sync" and advice that changes nothing is + worse than none. It asks git through a **throwaway git dir outside the + folder**, so a refused `add` leaves no `.git` behind in a directory the + user never agreed to turn into a repo. * `watch [--interval 30] [--once]` — per folder, by mode. `mirror`: autocommit (`sync: `) → fetch → ff-pull if remote strictly ahead → push if local strictly ahead. `snapshot`: fetch → push a snapshot of the @@ -370,18 +379,31 @@ deleted it again, `DELETE …/tokens/{id}` returning 204 under basic auth): force, never merge** — the same policy as the mesh — **but the backup still happens**, because divergence is when work is most at risk. Retention pruning runs at most hourly. SIGTERM-clean. -* `snapshots [--limit 20]` — restore points, newest first. Read from - the **remote**, not a local cache: the feature exists for the case where - this machine is gone. +* `snapshots [--limit 20]` — restore points, newest first, **across + every device**, with the device that took each one. Read from the + **remote**, not a local cache: the feature exists for the case where this + machine is gone. + + The three scopes differ deliberately. Writing is device-scoped, so two + machines never overwrite each other. Pruning is device-scoped, so machine A + never applies its clock to machine B's refs. **Reading is not scoped** — a + replacement laptop has a new id, and scoping the read to it would print + "no restore points yet" while the backups sit on the forge. That defect + was live in the first draft and is now pinned by a test that restores a + dead machine's work from a fresh clone. * `restore --at [--into DIR]` — materialise one restore - point into a new directory; refuses a non-empty destination. + point into a new directory; refuses a non-empty destination. Accepts what + `snapshots` printed in either mode, including a mirror-mode `%cI` + timestamp. Two commits inside the same second share that timestamp, so an + ambiguous `--at` is **refused with the candidate ids** rather than + resolved by guessing. * `status` — table of linked folders, mode, last sync, divergence flags. * Run as a daemon on macOS with `client/launchd/ai.granthi.sync.plist` (edit the script path, then `launchctl bootstrap gui/$UID `). ## Tests -* `python3 -m unittest discover -s tests` — 143 tests. The v1.2 additions +* `python3 -m unittest discover -s tests` — 153 tests. The v1.2 additions cover: a snapshot capturing uncommitted work while HEAD, the index and the working tree stay byte-identical; snapshots landing outside `refs/heads`; an unchanged tree not being re-pushed; a diverged folder still being backed diff --git a/client/granthi_sync_client.py b/client/granthi_sync_client.py index 953ee57..aed0d4f 100644 --- a/client/granthi_sync_client.py +++ b/client/granthi_sync_client.py @@ -354,25 +354,41 @@ def _remember_snapshot_tree(folder, dev, tree): pass # a lost marker costs one redundant push, never correctness -def list_snapshots(folder, dev, remote="granthi"): - """Backup refs for this device on the remote, newest first. +def list_snapshots(folder, dev=None, remote="granthi"): + """Backup refs on the remote, newest first. Read from the REMOTE, not a local cache: the point of the feature is surviving the loss of this machine, so what the server holds is the answer that matters. + + `dev=None` means EVERY device, and that is what the read paths pass. + The three scopes differ on purpose: + + * writing is device-scoped, so two machines never overwrite each + other's snapshots; + * pruning is device-scoped, so machine A never applies its own clock + and retention to machine B's refs; + * reading is NOT scoped, because the case this feature exists for is + "the laptop died". A new machine has a new id, and scoping the read + to it would show an empty list while the backups sit on the forge. """ - rc, out = git(folder, "ls-remote", remote, f"{BACKUP_NS}/{dev}/*", - check=False) + pattern = f"{BACKUP_NS}/{dev}/*" if dev else f"{BACKUP_NS}/*" + rc, out = git(folder, "ls-remote", remote, pattern, check=False) if rc != 0: return [] + prefix = f"{BACKUP_NS}/{dev}/" if dev else f"{BACKUP_NS}/" snaps = [] for line in out.splitlines(): sha, _, ref = line.partition("\t") ref = ref.strip() - if not ref.startswith(f"{BACKUP_NS}/{dev}/"): + if not ref.startswith(prefix): continue - snaps.append({"sha": sha.strip(), "ref": ref, - "ts": ref.rsplit("/", 1)[-1]}) + rest = ref[len(f"{BACKUP_NS}/"):] + device, _, ts = rest.partition("/") + if not ts: + continue + snaps.append({"sha": sha.strip(), "ref": ref, "ts": ts, + "device": device}) return sorted(snaps, key=lambda s: s["ts"], reverse=True) @@ -831,21 +847,45 @@ def _get_all(cfg, args): """ repos, truncated = list_repos(cfg) base = os.path.abspath(args.into or ".") - linked = set(cfg.get("folders", {})) + folders = cfg.get("folders", {}) + # full_name already living at a path, so "already here" can be told apart + # from "a different repo wants that same path". + owner_of = {os.path.abspath(p): m.get("full_name") + for p, m in folders.items()} cloned = skipped = failed = 0 for repo in sorted(repos, key=lambda r: r.get("full_name") or ""): full_name = repo.get("full_name") if not full_name: continue - dest = os.path.join(base, full_name.rsplit("/", 1)[-1]) - if dest in linked or (os.path.exists(dest) and os.listdir(dest)): + owner, _, name = full_name.rpartition("/") + dest = os.path.join(base, name) + if owner_of.get(dest) == full_name: log(f"skip {full_name}: already present at {dest}") skipped += 1 continue + if dest in owner_of or (os.path.exists(dest) and os.listdir(dest)): + # alice/notes and bob/notes both want /notes. Reporting the + # second as "already here" would be a silent collision -- the user + # would believe they had pulled both. + taken_by = owner_of.get(dest) + qualified = os.path.join(base, f"{owner}-{name}" if owner else name) + if owner_of.get(qualified) == full_name or ( + os.path.exists(qualified) and os.listdir(qualified)): + log(f"skip {full_name}: already present at {qualified}") + skipped += 1 + continue + log(f"name clash on {dest}" + + (f" (held by {taken_by})" if taken_by else "") + + f": cloning {full_name} to {qualified} instead") + dest = qualified try: clone_one(cfg, full_name, dest, args.mode) + owner_of[os.path.abspath(dest)] = full_name cloned += 1 - except SystemExit as e: # one bad repo must not abandon the rest + except (SystemExit, RuntimeError) as e: + # One bad repo must not abandon the rest. clone_one raises + # SystemExit for a refused destination and RuntimeError from any + # failing git call -- both are one repo's problem, not the run's. log(f"FAILED {full_name}: {e}") failed += 1 log(f"get --all: {cloned} cloned, {skipped} already here, {failed} failed") @@ -857,12 +897,65 @@ def _get_all(cfg, args): def measure_folder(folder, max_files=ADD_MAX_FILES): - """(files, bytes) below .git, stopping once max_files is exceeded. + """(files, bytes) that would actually sync, stopping past max_files. + + Ignore rules are honoured, because the guard's own advice is "add a + .gitignore for what should not sync" -- and advice that changes nothing + is worse than no advice. git is asked with a THROWAWAY git dir outside + the folder, so a refused `add` leaves no .git behind in a directory the + user never agreed to turn into a repo. Bounded on purpose: the caller only needs to know whether the folder is surprisingly big, and walking a 2 TB drive to answer that would be the same mistake as syncing it. """ + listing = _git_would_sync(folder) + if listing is None: + return _walk_folder(folder, max_files) + files = total = 0 + for rel in listing: + path = os.path.join(folder, rel) + try: + if os.path.islink(path): + continue + total += os.path.getsize(path) + except OSError: + continue + files += 1 + if files > max_files: + break + return files, total + + +def _git_would_sync(folder): + """Paths git would take from `folder`, honouring .gitignore. None if git + could not answer, so the caller falls back to a plain walk.""" + scratch = tempfile.mkdtemp(prefix="granthi-measure-") + try: + git_dir = os.path.join(scratch, "git") + init = subprocess.run(["git", "init", "-q", "--bare", git_dir], + capture_output=True, text=True) + if init.returncode != 0: + return None + proc = subprocess.run( + ["git", f"--git-dir={git_dir}", f"--work-tree={folder}", + "status", "--porcelain", "--untracked-files=all", + "--ignored=no", "-z"], + capture_output=True, text=True) + if proc.returncode != 0: + return None + out = [] + for entry in proc.stdout.split("\0"): + if len(entry) > 3: + out.append(entry[3:]) + return out + except OSError: + return None + finally: + subprocess.run(["rm", "-rf", scratch], capture_output=True) + + +def _walk_folder(folder, max_files): files = total = 0 for root, dirs, names in os.walk(folder): dirs[:] = [d for d in dirs if d != ".git"] @@ -989,16 +1082,15 @@ def _sigterm(signum, frame): log(f"signal {signum} received; finishing current pass then exiting") -_LAST_PRUNE = 0.0 - - def watch_pass(now=None): - global _LAST_PRUNE cfg = load_config() dev = device_id(cfg) # Pruning talks to the remote for every folder, so it runs on a slow - # clock of its own rather than on every 30s pass. - due = (now or time.time()) - _LAST_PRUNE >= PRUNE_EVERY_SECONDS + # clock of its own rather than on every 30s pass. The clock is PERSISTED: + # `watch --once` under launchd or cron is a fresh process every time, and + # an in-memory timer would make "at most hourly" mean "every run". + now = now or time.time() + due = now - float(cfg.get("last_prune") or 0) >= PRUNE_EVERY_SECONDS for folder, meta in sorted(cfg.get("folders", {}).items()): if not os.path.isdir(folder): log(f"SKIP {folder}: missing") @@ -1024,7 +1116,7 @@ def watch_pass(now=None): # Retention failing must never stop the backup itself. log(f"prune failed in {folder} (backups unaffected): {e}") if due: - _LAST_PRUNE = now or time.time() + cfg["last_prune"] = now save_config(cfg) @@ -1058,7 +1150,7 @@ def cmd_snapshots(args): """Restore points, newest first. Both modes answer the same question.""" cfg = require_linked(load_config()) folder, meta = _folder_meta(cfg, args.folder) - dev = device_id(cfg) + this_dev = cfg.get("device_id") if meta.get("mode", "mirror") == "mirror": rc, out = git(folder, "log", "--format=%H\t%cI\t%s", "-n", str(args.limit), check=False) @@ -1070,13 +1162,16 @@ def cmd_snapshots(args): sha, ts, subject = (line.split("\t") + ["", ""])[:3] print(f" {ts} {sha[:12]} {subject}") return 0 - snaps = list_snapshots(folder, dev) + # Every device's backups, not just this one's -- see list_snapshots. + snaps = list_snapshots(folder) if not snaps: print("no restore points yet (nothing uncommitted has been backed up)") return 0 - print(f"restore points for {folder} (mode snapshot, device {dev[:8]}):") + print(f"restore points for {folder} (mode snapshot):") for snap in snaps[:args.limit]: - print(f" {snap['ts']} {snap['sha'][:12]}") + here = " (this device)" if snap["device"] == this_dev else "" + print(f" {snap['ts']} {snap['sha'][:12]} " + f"device {snap['device'][:8]}{here}") if len(snaps) > args.limit: print(f" ... {len(snaps) - args.limit} older (use --limit)") return 0 @@ -1091,15 +1186,36 @@ def cmd_restore(args): """ cfg = require_linked(load_config()) folder, meta = _folder_meta(cfg, args.folder) - dev = device_id(cfg) target = args.at sha = None if meta.get("mode", "mirror") == "snapshot": - for snap in list_snapshots(folder, dev): + # Across ALL devices: restoring is exactly the moment the device that + # took the backup may no longer exist. + for snap in list_snapshots(folder): if snap["ts"] == target or snap["sha"].startswith(target): sha = snap["sha"] git(folder, "fetch", "granthi", f"{snap['ref']}") break + else: + # Mirror mode lists commits by their %cI timestamp, and that first + # column is what a person copies. It is not a commit-ish, so match it + # against the log before falling through to rev-parse -- otherwise + # `restore --at ` fails and sends the + # user back to `snapshots`, which prints the same thing again. + rc, out = git(folder, "log", "--format=%H\t%cI", check=False) + if rc == 0: + hits = [line.partition("\t")[0] for line in out.splitlines() + if line.partition("\t")[2] == target] + if len(hits) > 1: + # Two commits in the same second share a %cI. Silently taking + # the newest would restore something the user did not pick. + listed = "\n".join(f" {h[:12]}" for h in hits) + raise SystemExit( + f"{target} matches {len(hits)} restore points in " + f"{folder}. Re-run with one of these ids instead:\n" + f"{listed}") + if hits: + sha = hits[0] if sha is None: rc, resolved = git(folder, "rev-parse", "--verify", "--quiet", f"{target}^{{commit}}", check=False) @@ -1108,8 +1224,10 @@ def cmd_restore(args): f"no restore point {target!r} for {folder} " f"(see: granthi-sync snapshots {folder})") sha = resolved.strip() + # A %cI timestamp carries ':' and '+', which make an awkward folder name. + suffix = re.sub(r"[^A-Za-z0-9]+", "-", target).strip("-") or sha[:12] dest = os.path.abspath( - args.into or f"{folder.rstrip(os.sep)}-restore-{target}") + args.into or f"{folder.rstrip(os.sep)}-restore-{suffix}") if os.path.exists(dest) and os.listdir(dest): raise SystemExit(f"refusing to restore over a non-empty path: {dest}") os.makedirs(dest, exist_ok=True) diff --git a/tests/test_client.py b/tests/test_client.py index d360328..9352ddd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -10,6 +10,7 @@ import shutil import subprocess import sys import tempfile +import time import unittest from unittest import mock @@ -578,8 +579,45 @@ class TestSnapshots(GitScenarioBase): ref = client.push_snapshot(self.local, self.DEV) snaps = client.list_snapshots(self.local, self.DEV) self.assertEqual([s["ref"] for s in snaps], [ref]) + # writing is device-scoped: the ref carries THIS device's id, so two + # machines cannot overwrite each other + self.assertEqual(snaps[0]["device"], self.DEV) self.assertEqual(client.list_snapshots(self.local, "someone-else"), []) + def test_a_new_machine_can_see_the_dead_machine_s_backups(self): + """The case the whole feature exists for. Reads must NOT be scoped to + this device's id -- a replacement laptop has a new id, and scoping + would show an empty list while the backups sit on the forge.""" + self.write(self.local, "a.txt", "committed") + run_git(self.local, "add", "-A") + run_git(self.local, "commit", "-m", "c") + run_git(self.local, "push", "-u", "granthi", "main") + self.write(self.local, "a.txt", "PRECIOUS UNCOMMITTED WORK") + ref = client.push_snapshot(self.local, "laptop-that-died") + + # a fresh clone standing in for the replacement machine + newbox = os.path.join(self.tmp, "newbox") + subprocess.run(["git", "clone", "-q", "--origin", "granthi", + self.bare, newbox], check=True, capture_output=True, + env=GIT_ENV) + client.save_config({"gitea_base": self.tmp, "login": "alice", + "token": "t", "device_id": "brand-new-laptop", + "folders": {newbox: {"name": "cloud", + "full_name": "alice/cloud", + "branch": "main", + "mode": "snapshot"}}}) + + seen = client.list_snapshots(newbox) + self.assertEqual([s["ref"] for s in seen], [ref]) + self.assertEqual(seen[0]["device"], "laptop-that-died") + + # and it can actually restore it + dest = os.path.join(self.tmp, "recovered") + client.cmd_restore(argparse.Namespace(folder=newbox, at=seen[0]["ts"], + into=dest)) + with open(os.path.join(dest, "a.txt")) as f: + self.assertEqual(f.read(), "PRECIOUS UNCOMMITTED WORK") + class TestRetention(unittest.TestCase): """Retention is what keeps 30-second backups from being a disk leak -- @@ -900,5 +938,169 @@ class TestCredentialHelperIsolation(GitScenarioBase): self.assertNotIn("stale-token", out.stdout) +class TestMirrorRestore(GitScenarioBase): + """Mirror is the default for a plain folder, so its restore path is the + one most people will use -- and the timestamp `snapshots` prints has to + be a timestamp `restore` accepts, or the user just loops.""" + + def setUp(self): + super().setUp() + client.save_config({"gitea_base": self.tmp, "login": "alice", + "token": "t", "device_id": "d1", + "folders": {self.local: { + "name": "cloud", "full_name": "alice/cloud", + "branch": "main", "mode": "mirror"}}}) + + def test_restore_accepts_the_timestamp_snapshots_printed(self): + self.write(self.local, "a.txt", "the version I want back") + client.sync_folder(self.local, mode="mirror") + time.sleep(1.1) # %cI has one-second resolution + self.write(self.local, "a.txt", "later junk") + client.sync_folder(self.local, mode="mirror") + + with mock.patch("sys.stdout", new_callable=io.StringIO) as out: + client.cmd_snapshots(argparse.Namespace(folder=self.local, + limit=20)) + listed = [l.split() for l in out.getvalue().splitlines() + if l.startswith(" ")] + wanted_ts = listed[-1][0] # first column of the oldest entry + + dest = os.path.join(self.tmp, "mirror-restore") + client.cmd_restore(argparse.Namespace(folder=self.local, + at=wanted_ts, into=dest)) + with open(os.path.join(dest, "a.txt")) as f: + self.assertEqual(f.read(), "the version I want back") + + def test_two_commits_in_the_same_second_are_refused_not_guessed(self): + """%cI has one-second resolution. Picking one silently would restore + something the user did not choose.""" + self.write(self.local, "a.txt", "first") + client.sync_folder(self.local, mode="mirror") + self.write(self.local, "a.txt", "second") + client.sync_folder(self.local, mode="mirror") + stamps = run_git(self.local, "log", "--format=%cI").splitlines() + if len(set(stamps)) != 1: + self.skipTest("commits did not land in the same second") + with self.assertRaises(SystemExit) as caught: + client.cmd_restore(argparse.Namespace(folder=self.local, + at=stamps[0], into=None)) + self.assertIn("matches 2 restore points", str(caught.exception)) + + def test_restore_still_accepts_a_sha(self): + self.write(self.local, "a.txt", "one") + client.sync_folder(self.local, mode="mirror") + sha = run_git(self.local, "rev-parse", "HEAD") + dest = os.path.join(self.tmp, "by-sha") + client.cmd_restore(argparse.Namespace(folder=self.local, at=sha, + into=dest)) + self.assertTrue(os.path.exists(os.path.join(dest, "a.txt"))) + + def test_default_destination_is_a_usable_path(self): + self.write(self.local, "a.txt", "one") + client.sync_folder(self.local, mode="mirror") + with mock.patch("sys.stdout", new_callable=io.StringIO) as out: + client.cmd_snapshots(argparse.Namespace(folder=self.local, + limit=5)) + ts = [l.split() for l in out.getvalue().splitlines() + if l.startswith(" ")][0][0] + client.cmd_restore(argparse.Namespace(folder=self.local, at=ts, + into=None)) + made = [d for d in os.listdir(self.tmp) if d.startswith("local-restore-")] + self.assertEqual(len(made), 1, made) + self.assertNotIn(":", made[0]) + + +class TestMeasureHonoursGitignore(GitScenarioBase): + """The guard tells people to add a .gitignore. That advice has to work.""" + + def test_ignored_files_are_not_counted(self): + os.makedirs(os.path.join(self.local, "bulkdata")) + for i in range(30): + self.write(self.local, f"bulkdata/x{i}.bin", "y" * 100) + self.write(self.local, "real.txt", "mine") + before, _ = client.measure_folder(self.local) + self.write(self.local, ".gitignore", "bulkdata/\n") + after, _ = client.measure_folder(self.local) + self.assertGreater(before, 30) + self.assertEqual(after, 2) # real.txt + .gitignore + + def test_measure_leaves_no_git_dir_behind(self): + plain = os.path.join(self.tmp, "untouched") + os.makedirs(plain) + self.write(plain, "a.txt", "x") + client.measure_folder(plain) + self.assertEqual(os.listdir(plain), ["a.txt"]) + + +class TestGetAllRobustness(GitScenarioBase): + def setUp(self): + super().setUp() + self.forge = os.path.join(self.tmp, "forge") + for full in ("alice/notes", "bob/notes"): + path = os.path.join(self.forge, full + ".git") + os.makedirs(os.path.dirname(path), exist_ok=True) + subprocess.run(["git", "init", "-q", "--bare", "-b", "main", path], + check=True, capture_output=True, env=GIT_ENV) + seed = os.path.join(self.tmp, "seed-" + full.replace("/", "-")) + subprocess.run(["git", "clone", "-q", path, seed], check=True, + capture_output=True, env=GIT_ENV) + run_git(seed, "config", "user.name", "s") + run_git(seed, "config", "user.email", "s@s") + self.write(seed, "who.txt", full) + run_git(seed, "add", "-A") + run_git(seed, "commit", "-m", "seed") + run_git(seed, "push", "-q", "origin", "main") + client.save_config({"gitea_base": self.forge, "login": "alice", + "token": "t", "folders": {}}) + self.repos = [{"name": "notes", "full_name": "alice/notes"}, + {"name": "notes", "full_name": "bob/notes"}] + + def test_same_named_repos_from_two_owners_both_land(self): + into = os.path.join(self.tmp, "ws") + os.makedirs(into) + with mock.patch.object(client, "list_repos", + lambda c: (self.repos, False)), \ + mock.patch("sys.stdout", new_callable=io.StringIO) as out: + client.cmd_get(get_ns(all=True, into=into)) + self.assertIn("name clash", out.getvalue()) + with open(os.path.join(into, "notes", "who.txt")) as f: + self.assertEqual(f.read(), "alice/notes") + with open(os.path.join(into, "bob-notes", "who.txt")) as f: + self.assertEqual(f.read(), "bob/notes") + self.assertEqual(len(client.load_config()["folders"]), 2) + + def test_a_git_failure_on_one_repo_does_not_abandon_the_rest(self): + into = os.path.join(self.tmp, "ws2") + os.makedirs(into) + real_clone = client.clone_one + + def flaky(cfg, full_name, dest, mode=None): + if full_name == "alice/notes": + raise RuntimeError("git clone failed: pretend network blip") + return real_clone(cfg, full_name, dest, mode) + + with mock.patch.object(client, "list_repos", + lambda c: (self.repos, False)), \ + mock.patch.object(client, "clone_one", flaky), \ + mock.patch("sys.stdout", new_callable=io.StringIO) as out: + rc = client.cmd_get(get_ns(all=True, into=into)) + self.assertEqual(rc, 1) # reported, not hidden + self.assertIn("FAILED alice/notes", out.getvalue()) + self.assertTrue(os.path.exists(os.path.join(into, "notes", "who.txt"))) + + +class TestPruneClockIsPersisted(GitScenarioBase): + def test_watch_once_does_not_prune_every_run(self): + client.save_config({"gitea_base": self.tmp, "login": "alice", + "token": "t", "device_id": "d1", "folders": {}}) + with mock.patch.object(client, "prune_snapshots") as pruner: + client.watch_pass(now=1_000_000.0) + self.assertEqual(client.load_config()["last_prune"], 1_000_000.0) + client.watch_pass(now=1_000_060.0) # a minute later: not due + client.watch_pass(now=1_003_700.0) # an hour later: due again + self.assertEqual(client.load_config()["last_prune"], 1_003_700.0) + self.assertEqual(pruner.call_count, 0) # no snapshot folders linked + + if __name__ == "__main__": unittest.main()