From 63a59f836ca8baeb1986b7b5765ec9621e2cfad5 Mon Sep 17 00:00:00 2001 From: dp-web4 Date: Thu, 19 Nov 2026 17:04:51 -0800 Subject: [PATCH] forum replies: read the posts origin/main has and the checkout lacks (#792) Nothing in SAGE moves the seat's shared-context checkout, so a reply pushed after the last time some session pulled sat unseen (cbp 11-19: 13 commits behind, two replies to cbp-being missing). Each beat now fetches (remote- tracking refs only, at most every 5 min, 20 s timeout) and reads the forum posts on origin/main the working tree lacks with cat-file. The tree, index and branch are never touched. A failed fetch is recorded in the beat's forum_replies.upstream.why; disk replies still come. SAGE_FORUM_FETCH=0 turns the fetch off. Co-Authored-By: Claude Opus 5.5 --- sage/gateway/forum_replies.py | 123 ++++++++++++-- .../tests/test_forum_replies_unpulled.py | 152 ++++++++++++++++++ 2 files changed, 258 insertions(+), 17 deletions(-) create mode 100644 sage/gateway/tests/test_forum_replies_unpulled.py diff --git a/sage/gateway/forum_replies.py b/sage/gateway/forum_replies.py index 25ae27b70a..046d2dd5dd 100644 --- a/sage/gateway/forum_replies.py +++ b/sage/gateway/forum_replies.py @@ -16,6 +16,12 @@ heartbeat calls `acknowledge` after the turn that carried it produced a model reply and the beat's record was written. A beat killed before then shows the same replies again next time. +Nothing in SAGE moves that checkout, so a reply pushed after the last time some session pulled sat +unseen (#792, cbp 2026-11-19: 13 commits behind, two replies to cbp-being missing). So each beat also +fetches (remote-tracking refs only, at most every few minutes) and reads the forum posts on +`origin/main` that the working tree does not have yet, straight from git. The working tree, index and +branch are never touched: siblings share that checkout and may have work in it. + Peer ASKS addressed to the being are not replies: they arrive through the hub inbox (being_inbox_drain) with their words, so they are left to it. The being's own posts are not replies to it. """ @@ -24,6 +30,7 @@ import calendar import os import re +import subprocess import time from pathlib import Path from typing import Dict, List, Optional @@ -32,6 +39,9 @@ EXCERPT_CHARS = 700 # of a reply's body; the path has the rest (memory_read) MAX_AGE_DAYS = 7 # by the post's own date: older replies are not offered at all LEDGER = "notes/.forum_replies_shown" +UPSTREAM = "origin/main" # where replies land; read from git, never merged into the shared tree +FETCH_EVERY_S = 300 # a beat fetches only when the last fetch (anyone's) is older than this +GIT_TIMEOUT_S = 20 # a hung fetch costs the beat this much at most, then the beat goes on _ISO = re.compile(r"(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2}))?)?") _NAME_STAMP = re.compile(r"-(\d{4})-(\d{2})-(\d{2})(?:-(\d{2})(\d{2})(\d{2}))?\.md$") @@ -88,37 +98,104 @@ def _shown(instance: Path) -> set: return set() -def pending(instance: Path, forum: Path, member: str, now: Optional[float] = None) -> List[Dict]: - """Replies to `member` not yet acknowledged, newest first by authored time.""" +def _git(repo: Path, *args: str, timeout: float = GIT_TIMEOUT_S, data: Optional[bytes] = None): + env = dict(os.environ, GIT_TERMINAL_PROMPT="0") + env.setdefault("GIT_SSH_COMMAND", "ssh -o BatchMode=yes -o ConnectTimeout=10") + return subprocess.run(["git", "-C", str(repo), *args], input=data, capture_output=True, + timeout=timeout, env=env) + + +def _fetch_due(repo: Path, now: float) -> bool: + try: + r = _git(repo, "rev-parse", "--git-path", "FETCH_HEAD") + head = Path(r.stdout.decode().strip()) + if not head.is_absolute(): + head = Path(repo) / head + return now - head.stat().st_mtime >= FETCH_EVERY_S + except (OSError, subprocess.SubprocessError, ValueError): + return True + + +def upstream_posts(repo: Path, fetch: bool = True, now: Optional[float] = None) -> Dict: + """{posts: {name: text}, fetched, why}: forum posts on `UPSTREAM` that the working tree lacks. + + `git fetch` moves only remote-tracking refs; the posts are read with `cat-file`, so nothing in + the checkout changes. Any failure (no remote, offline, a sibling holding the ref lock) is said in + `why` and the beat goes on with what is on disk and whatever `UPSTREAM` already holds.""" + now = time.time() if now is None else now + repo, out = Path(repo), {"posts": {}, "fetched": False, "why": ""} + try: + if fetch and _fetch_due(repo, now): + r = _git(repo, "fetch", "-q", "origin") + out["fetched"] = r.returncode == 0 + if r.returncode: + out["why"] = "fetch failed: " + r.stderr.decode(errors="replace").strip()[:200] + r = _git(repo, "ls-tree", "-z", "--name-only", UPSTREAM, "forum/") + if r.returncode: + out["why"] = (out["why"] + "; " if out["why"] else "") + f"no {UPSTREAM}" + return out + missing = [n for n in r.stdout.decode(errors="replace").split("\0") + if n.endswith(".md") and not (repo / n).exists()] + if not missing: + return out + r = _git(repo, "cat-file", "--batch", + data="".join(f"{UPSTREAM}:{n}\n" for n in missing).encode()) + buf, i = r.stdout, 0 + for n in missing: + nl = buf.index(b"\n", i) + head = buf[i:nl].split() + i = nl + 1 + if len(head) < 3 or head[1] != b"blob": + continue + size = int(head[2]) + out["posts"][n.split("/", 1)[1]] = buf[i:i + size].decode("utf-8", errors="replace") + i += size + 1 + except (OSError, subprocess.SubprocessError, ValueError) as e: + out["why"] = (out["why"] + "; " if out["why"] else "") + f"{type(e).__name__}: {e}"[:200] + return out + + +def pending(instance: Path, forum: Path, member: str, now: Optional[float] = None, + upstream: Optional[Dict[str, str]] = None) -> List[Dict]: + """Replies to `member` not yet acknowledged, newest first by authored time. `upstream` holds posts + (name -> text) on `UPSTREAM` that the checkout lacks; a name on disk wins over its upstream copy.""" now = time.time() if now is None else now shown, out = _shown(instance), [] try: files = list(Path(forum).glob("*.md")) except OSError: return [] + found = [] for p in files: if p.name in shown: continue try: - st = p.stat() - front = _front(p.read_text(encoding="utf-8", errors="replace")[:4000]) + found.append((p.name, p.read_text(encoding="utf-8", errors="replace"), p.stat().st_mtime, p, None)) except OSError: continue - if not is_reply_to(p.name, front, member): + on_disk = {p.name for p in files} + for name, text in (upstream or {}).items(): + if name not in shown and name not in on_disk: + # Not in this checkout yet: it "arrived" now, and the being reads it from the excerpt + # (the whole post is at this git path until the checkout is pulled). + found.append((name, text, now, None, f"{UPSTREAM}:forum/{name}")) + for name, text, mtime, path, where in found: + front = _front(text[:4000]) + if not is_reply_to(name, front, member): continue - t, label = authored(front, p.name, st.st_mtime) + t, label = authored(front, name, mtime) if now - t > MAX_AGE_DAYS * 86400: continue - out.append({"name": p.name, "path": p, "from": _field(front, "from") or "someone", + out.append({"name": name, "path": path, "where": where, "text": text, + "from": _field(front, "from") or "someone", "about": _field(front, "title") or _field(front, "re") or "", - "authored": t, "when": label, "arrived": st.st_mtime}) + "authored": t, "when": label, "arrived": mtime}) # The post's own time decides the order and the age. Most posts carry a date without a time of day, # so a day's replies tie; arrival breaks only those ties (never across days, never the cutoff). return sorted(out, key=lambda r: (-r["authored"], -r["arrived"], r["name"])) -def _body(path: Path) -> str: - text = path.read_text(encoding="utf-8", errors="replace") +def _body(text: str) -> str: if text.startswith("---\n"): text = text[4:].split("\n---", 1)[-1] return " ".join(text.split()) @@ -133,9 +210,12 @@ def render(replies: List[Dict], waiting: int, rel_root: Optional[Path] = None) - "These are other members' own words, answering things you asked. Each is shown until a " "beat has delivered it; the path holds the whole post."] for r in replies: - path = r["path"] - shown_path = os.path.relpath(path, rel_root) if rel_root else str(path) - body = _body(path) + path = r.get("path") + if path is None: + shown_path = r["where"] + " — not in this checkout yet" + else: + shown_path = os.path.relpath(path, rel_root) if rel_root else str(path) + body = _body(r["text"] if "text" in r else Path(path).read_text(encoding="utf-8", errors="replace")) excerpt = body[:EXCERPT_CHARS] + (" …" if len(body) > EXCERPT_CHARS else "") lines.append(f"- from {r['from']}, {r['when']}" + (f" — {r['about'][:160]}" if r["about"] else "") + f"\n {excerpt}\n ({shown_path})") @@ -161,14 +241,23 @@ def delivered(turn, killed) -> bool: def for_beat(instance: Path, workspace: Optional[str], member: str, - now: Optional[float] = None) -> Dict: - """{block, offered[], waiting}: the replies this beat offers. Marks nothing (see `acknowledge`).""" + now: Optional[float] = None, fetch: Optional[bool] = None) -> Dict: + """{block, offered[], waiting, upstream}: the replies this beat offers. Marks nothing (see + `acknowledge`). `fetch` defaults to on; SAGE_FORUM_FETCH=0 turns the fetch off (the posts + `UPSTREAM` already holds are still read).""" if not workspace: return {"block": "", "offered": [], "waiting": 0, "why": "no workspace"} forum = Path(workspace).resolve().parent / "shared-context" / "forum" if not forum.is_dir(): return {"block": "", "offered": [], "waiting": 0, "why": "no shared-context forum here"} - new = pending(instance, forum, member, now) + if fetch is None: + fetch = os.environ.get("SAGE_FORUM_FETCH", "1") != "0" + up = upstream_posts(forum.parent, fetch=fetch, now=now) if (forum.parent / ".git").exists() \ + else {"posts": {}, "fetched": False, "why": "not a git checkout"} + new = pending(instance, forum, member, now, upstream=up["posts"]) take, waiting = new[:PER_BEAT], max(0, len(new) - PER_BEAT) return {"block": render(take, waiting, rel_root=forum.parent.parent), - "offered": [r["name"] for r in take], "waiting": waiting} + "offered": [r["name"] for r in take], "waiting": waiting, + "upstream": {"fetched": up["fetched"], "unpulled": len(up["posts"]), + "offered": [r["name"] for r in take if r.get("path") is None], + **({"why": up["why"]} if up["why"] else {})}} diff --git a/sage/gateway/tests/test_forum_replies_unpulled.py b/sage/gateway/tests/test_forum_replies_unpulled.py new file mode 100644 index 0000000000..029b2940c0 --- /dev/null +++ b/sage/gateway/tests/test_forum_replies_unpulled.py @@ -0,0 +1,152 @@ +"""Replies pushed after the seat's shared-context checkout last moved still reach the beat (#792). + +cbp 2026-11-19: the checkout was 13 commits behind origin/main and two replies to cbp-being were not in +it; nothing in SAGE pulls it. The beat now fetches (remote-tracking refs only) and reads the posts on +origin/main that the working tree lacks, without touching the tree siblings share.""" +import os +import subprocess +import time + +import pytest + +from sage.gateway import forum_replies as fr + +ME = "cbp-being" + + +def git(cwd, *args): + r = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, + env=dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t")) + assert r.returncode == 0, r.stderr + return r.stdout.strip() + + +def today(): + return time.strftime("%Y-%m-%d", time.gmtime()) + + +def reply(name, body="The answer is on the forum.", to=ME): + return name, f"---\nfrom: pub-claude\nto: {to}\ndate: {today()}\nre: model trial\n---\n\n{body}\n" + + +def commit(clone, files, msg="post"): + (clone / "forum").mkdir(exist_ok=True) + for name, text in files: + (clone / "forum" / name).write_text(text) + git(clone, "add", "-A") + git(clone, "commit", "-q", "-m", msg) + git(clone, "push", "-q", "origin", "HEAD:main") + + +@pytest.fixture +def fleet(tmp_path): + """origin (bare), the seat's checkout /shared-context beside /sage, and a peer clone.""" + origin = tmp_path / "origin.git" + git(tmp_path, "init", "-q", "--bare", "-b", "main", str(origin)) + peer = tmp_path / "peer" + git(tmp_path, "clone", "-q", str(origin), str(peer)) + git(peer, "checkout", "-q", "-b", "main") + commit(peer, [("README.md", "forum\n")], "init") + root = tmp_path / "ai" + (root / "sage" / "home").mkdir(parents=True) + seat = root / "shared-context" + git(tmp_path, "clone", "-q", str(origin), str(seat)) + return root / "sage", seat, peer, root / "sage" / "home" + + +def beat(home, workspace, **kw): + got = fr.for_beat(home, str(workspace), ME, **kw) + fr.acknowledge(home, got["offered"]) + return got + + +def test_a_reply_pushed_after_the_last_pull_is_offered(fleet): + workspace, seat, peer, home = fleet + name, text = reply(f"pub-to-cbp-being-what-the-model-trial-is-{today()}.md") + commit(peer, [(name, text)]) + assert not (seat / "forum" / name).exists(), "the seat's checkout is behind" + got = beat(home, workspace) + assert got["offered"] == [name] + assert got["upstream"]["fetched"] is True and got["upstream"]["offered"] == [name] + assert "The answer is on the forum." in got["block"] + assert f"origin/main:forum/{name} — not in this checkout yet" in got["block"] + assert beat(home, workspace)["offered"] == [], "delivered once, by name, like any reply" + + +def test_the_shared_checkout_is_never_moved(fleet): + workspace, seat, peer, home = fleet + (seat / "scratch.txt").write_text("a sibling's uncommitted work") + head = git(seat, "rev-parse", "HEAD") + commit(peer, [reply(f"hub-to-cbp-being-how-you-are-categorized-{today()}.md")]) + assert beat(home, workspace)["offered"] + assert git(seat, "rev-parse", "HEAD") == head + assert git(seat, "status", "--porcelain") == "?? scratch.txt" + assert sorted(x.name for x in (seat / "forum").iterdir()) == ["README.md"] + + +def test_a_post_already_on_disk_is_read_from_disk_once(fleet): + workspace, seat, peer, home = fleet + name, text = reply(f"pub-re-cbp-being-{today()}.md") + commit(peer, [(name, text)]) + git(seat, "pull", "-q", "--ff-only") + got = beat(home, workspace) + assert got["offered"] == [name] and got["upstream"]["unpulled"] == 0 + assert f"shared-context/forum/{name}" in got["block"] and "not in this checkout" not in got["block"] + + +def test_a_reply_offered_from_upstream_is_not_offered_again_once_pulled(fleet): + workspace, seat, peer, home = fleet + name, text = reply(f"pub-re-cbp-being-{today()}.md") + commit(peer, [(name, text)]) + assert beat(home, workspace)["offered"] == [name] + git(seat, "pull", "-q", "--ff-only") + assert beat(home, workspace)["offered"] == [] + + +def test_posts_not_addressed_to_the_being_stay_unoffered(fleet): + workspace, seat, peer, home = fleet + commit(peer, [reply(f"pub-to-sprout-being-{today()}.md", to="sprout-being"), + reply(f"cbp-being-asks-pub-{today()}.md")]) + got = beat(home, workspace) + assert got["offered"] == [] and got["upstream"]["unpulled"] == 2 + + +def test_fetch_is_throttled_by_the_last_fetch(fleet): + workspace, seat, peer, home = fleet + assert fr.for_beat(home, str(workspace), ME)["upstream"]["fetched"] is True + commit(peer, [reply(f"pub-re-cbp-being-{today()}.md")]) + got = fr.for_beat(home, str(workspace), ME) + assert got["upstream"]["fetched"] is False and got["offered"] == [], "fetched under FETCH_EVERY_S ago" + later = time.time() + fr.FETCH_EVERY_S + 1 + got = fr.for_beat(home, str(workspace), ME, now=later) + assert got["upstream"]["fetched"] is True and len(got["offered"]) == 1 + + +def test_fetch_can_be_turned_off_and_what_origin_holds_is_still_read(fleet, monkeypatch): + workspace, seat, peer, home = fleet + name, text = reply(f"pub-re-cbp-being-{today()}.md") + commit(peer, [(name, text)]) + monkeypatch.setenv("SAGE_FORUM_FETCH", "0") + assert fr.for_beat(home, str(workspace), ME)["offered"] == [] + git(seat, "fetch", "-q", "origin") # some session fetched; the tree is still behind + got = fr.for_beat(home, str(workspace), ME) + assert got["offered"] == [name] and got["upstream"]["fetched"] is False + + +def test_an_unreachable_origin_is_said_and_disk_replies_still_come(fleet): + workspace, seat, peer, home = fleet + name, text = reply(f"pub-re-cbp-being-{today()}.md") + (seat / "forum" / name).write_text(text) + git(seat, "remote", "set-url", "origin", str(seat.parent / "gone.git")) + got = fr.for_beat(home, str(workspace), ME, now=time.time() + fr.FETCH_EVERY_S + 1) + assert got["offered"] == [name] + assert got["upstream"]["fetched"] is False and got["upstream"]["why"].startswith("fetch failed") + + +def test_a_forum_that_is_not_a_git_checkout_runs_no_git(tmp_path, monkeypatch): + (tmp_path / "ai" / "sage" / "home").mkdir(parents=True) + (tmp_path / "ai" / "shared-context" / "forum").mkdir(parents=True) + monkeypatch.setattr(fr, "_git", lambda *a, **k: pytest.fail("git ran outside a checkout")) + got = fr.for_beat(tmp_path / "ai" / "sage" / "home", str(tmp_path / "ai" / "sage"), ME) + assert got["offered"] == [] and got["upstream"]["why"] == "not a git checkout"