Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 106 additions & 17 deletions sage/gateway/forum_replies.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@
heartbeat calls `acknowledge` after the turn that carried it produced a model reply and the beat's
record was written. A beat killed before then shows the same replies again next time.

Nothing in SAGE moves that checkout, so a reply pushed after the last time some session pulled sat
unseen (#792, cbp 2026-11-19: 13 commits behind, two replies to cbp-being missing). So each beat also
fetches (remote-tracking refs only, at most every few minutes) and reads the forum posts on
`origin/main` that the working tree does not have yet, straight from git. The working tree, index and
branch are never touched: siblings share that checkout and may have work in it.

Peer ASKS addressed to the being are not replies: they arrive through the hub inbox (being_inbox_drain)
with their words, so they are left to it. The being's own posts are not replies to it.
"""
Expand All @@ -24,6 +30,7 @@
import calendar
import os
import re
import subprocess
import time
from pathlib import Path
from typing import Dict, List, Optional
Expand All @@ -32,6 +39,9 @@
EXCERPT_CHARS = 700 # of a reply's body; the path has the rest (memory_read)
MAX_AGE_DAYS = 7 # by the post's own date: older replies are not offered at all
LEDGER = "notes/.forum_replies_shown"
UPSTREAM = "origin/main" # where replies land; read from git, never merged into the shared tree
FETCH_EVERY_S = 300 # a beat fetches only when the last fetch (anyone's) is older than this
GIT_TIMEOUT_S = 20 # a hung fetch costs the beat this much at most, then the beat goes on

_ISO = re.compile(r"(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2}))?)?")
_NAME_STAMP = re.compile(r"-(\d{4})-(\d{2})-(\d{2})(?:-(\d{2})(\d{2})(\d{2}))?\.md$")
Expand Down Expand Up @@ -88,37 +98,104 @@ def _shown(instance: Path) -> set:
return set()


def pending(instance: Path, forum: Path, member: str, now: Optional[float] = None) -> List[Dict]:
"""Replies to `member` not yet acknowledged, newest first by authored time."""
def _git(repo: Path, *args: str, timeout: float = GIT_TIMEOUT_S, data: Optional[bytes] = None):
env = dict(os.environ, GIT_TERMINAL_PROMPT="0")
env.setdefault("GIT_SSH_COMMAND", "ssh -o BatchMode=yes -o ConnectTimeout=10")
return subprocess.run(["git", "-C", str(repo), *args], input=data, capture_output=True,
timeout=timeout, env=env)


def _fetch_due(repo: Path, now: float) -> bool:
try:
r = _git(repo, "rev-parse", "--git-path", "FETCH_HEAD")
head = Path(r.stdout.decode().strip())
if not head.is_absolute():
head = Path(repo) / head
return now - head.stat().st_mtime >= FETCH_EVERY_S
except (OSError, subprocess.SubprocessError, ValueError):
return True


def upstream_posts(repo: Path, fetch: bool = True, now: Optional[float] = None) -> Dict:
"""{posts: {name: text}, fetched, why}: forum posts on `UPSTREAM` that the working tree lacks.

`git fetch` moves only remote-tracking refs; the posts are read with `cat-file`, so nothing in
the checkout changes. Any failure (no remote, offline, a sibling holding the ref lock) is said in
`why` and the beat goes on with what is on disk and whatever `UPSTREAM` already holds."""
now = time.time() if now is None else now
repo, out = Path(repo), {"posts": {}, "fetched": False, "why": ""}
try:
if fetch and _fetch_due(repo, now):
r = _git(repo, "fetch", "-q", "origin")
out["fetched"] = r.returncode == 0
if r.returncode:
out["why"] = "fetch failed: " + r.stderr.decode(errors="replace").strip()[:200]
r = _git(repo, "ls-tree", "-z", "--name-only", UPSTREAM, "forum/")
if r.returncode:
out["why"] = (out["why"] + "; " if out["why"] else "") + f"no {UPSTREAM}"
return out
missing = [n for n in r.stdout.decode(errors="replace").split("\0")
if n.endswith(".md") and not (repo / n).exists()]
if not missing:
return out
r = _git(repo, "cat-file", "--batch",
data="".join(f"{UPSTREAM}:{n}\n" for n in missing).encode())
buf, i = r.stdout, 0
for n in missing:
nl = buf.index(b"\n", i)
head = buf[i:nl].split()
i = nl + 1
if len(head) < 3 or head[1] != b"blob":
continue
size = int(head[2])
out["posts"][n.split("/", 1)[1]] = buf[i:i + size].decode("utf-8", errors="replace")
i += size + 1
except (OSError, subprocess.SubprocessError, ValueError) as e:
out["why"] = (out["why"] + "; " if out["why"] else "") + f"{type(e).__name__}: {e}"[:200]
return out


def pending(instance: Path, forum: Path, member: str, now: Optional[float] = None,
upstream: Optional[Dict[str, str]] = None) -> List[Dict]:
"""Replies to `member` not yet acknowledged, newest first by authored time. `upstream` holds posts
(name -> text) on `UPSTREAM` that the checkout lacks; a name on disk wins over its upstream copy."""
now = time.time() if now is None else now
shown, out = _shown(instance), []
try:
files = list(Path(forum).glob("*.md"))
except OSError:
return []
found = []
for p in files:
if p.name in shown:
continue
try:
st = p.stat()
front = _front(p.read_text(encoding="utf-8", errors="replace")[:4000])
found.append((p.name, p.read_text(encoding="utf-8", errors="replace"), p.stat().st_mtime, p, None))
except OSError:
continue
if not is_reply_to(p.name, front, member):
on_disk = {p.name for p in files}
for name, text in (upstream or {}).items():
if name not in shown and name not in on_disk:
# Not in this checkout yet: it "arrived" now, and the being reads it from the excerpt
# (the whole post is at this git path until the checkout is pulled).
found.append((name, text, now, None, f"{UPSTREAM}:forum/{name}"))
for name, text, mtime, path, where in found:
front = _front(text[:4000])
if not is_reply_to(name, front, member):
continue
t, label = authored(front, p.name, st.st_mtime)
t, label = authored(front, name, mtime)
if now - t > MAX_AGE_DAYS * 86400:
continue
out.append({"name": p.name, "path": p, "from": _field(front, "from") or "someone",
out.append({"name": name, "path": path, "where": where, "text": text,
"from": _field(front, "from") or "someone",
"about": _field(front, "title") or _field(front, "re") or "",
"authored": t, "when": label, "arrived": st.st_mtime})
"authored": t, "when": label, "arrived": mtime})
# The post's own time decides the order and the age. Most posts carry a date without a time of day,
# so a day's replies tie; arrival breaks only those ties (never across days, never the cutoff).
return sorted(out, key=lambda r: (-r["authored"], -r["arrived"], r["name"]))


def _body(path: Path) -> str:
text = path.read_text(encoding="utf-8", errors="replace")
def _body(text: str) -> str:
if text.startswith("---\n"):
text = text[4:].split("\n---", 1)[-1]
return " ".join(text.split())
Expand All @@ -133,9 +210,12 @@ def render(replies: List[Dict], waiting: int, rel_root: Optional[Path] = None) -
"These are other members' own words, answering things you asked. Each is shown until a "
"beat has delivered it; the path holds the whole post."]
for r in replies:
path = r["path"]
shown_path = os.path.relpath(path, rel_root) if rel_root else str(path)
body = _body(path)
path = r.get("path")
if path is None:
shown_path = r["where"] + " — not in this checkout yet"
else:
shown_path = os.path.relpath(path, rel_root) if rel_root else str(path)
body = _body(r["text"] if "text" in r else Path(path).read_text(encoding="utf-8", errors="replace"))
excerpt = body[:EXCERPT_CHARS] + (" …" if len(body) > EXCERPT_CHARS else "")
lines.append(f"- from {r['from']}, {r['when']}" + (f" — {r['about'][:160]}" if r["about"] else "")
+ f"\n {excerpt}\n ({shown_path})")
Expand All @@ -161,14 +241,23 @@ def delivered(turn, killed) -> bool:


def for_beat(instance: Path, workspace: Optional[str], member: str,
now: Optional[float] = None) -> Dict:
"""{block, offered[], waiting}: the replies this beat offers. Marks nothing (see `acknowledge`)."""
now: Optional[float] = None, fetch: Optional[bool] = None) -> Dict:
"""{block, offered[], waiting, upstream}: the replies this beat offers. Marks nothing (see
`acknowledge`). `fetch` defaults to on; SAGE_FORUM_FETCH=0 turns the fetch off (the posts
`UPSTREAM` already holds are still read)."""
if not workspace:
return {"block": "", "offered": [], "waiting": 0, "why": "no workspace"}
forum = Path(workspace).resolve().parent / "shared-context" / "forum"
if not forum.is_dir():
return {"block": "", "offered": [], "waiting": 0, "why": "no shared-context forum here"}
new = pending(instance, forum, member, now)
if fetch is None:
fetch = os.environ.get("SAGE_FORUM_FETCH", "1") != "0"
up = upstream_posts(forum.parent, fetch=fetch, now=now) if (forum.parent / ".git").exists() \
else {"posts": {}, "fetched": False, "why": "not a git checkout"}
new = pending(instance, forum, member, now, upstream=up["posts"])
take, waiting = new[:PER_BEAT], max(0, len(new) - PER_BEAT)
return {"block": render(take, waiting, rel_root=forum.parent.parent),
"offered": [r["name"] for r in take], "waiting": waiting}
"offered": [r["name"] for r in take], "waiting": waiting,
"upstream": {"fetched": up["fetched"], "unpulled": len(up["posts"]),
"offered": [r["name"] for r in take if r.get("path") is None],
**({"why": up["why"]} if up["why"] else {})}}
152 changes: 152 additions & 0 deletions sage/gateway/tests/test_forum_replies_unpulled.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
"""Replies pushed after the seat's shared-context checkout last moved still reach the beat (#792).

cbp 2026-11-19: the checkout was 13 commits behind origin/main and two replies to cbp-being were not in
it; nothing in SAGE pulls it. The beat now fetches (remote-tracking refs only) and reads the posts on
origin/main that the working tree lacks, without touching the tree siblings share."""
import os
import subprocess
import time

import pytest

from sage.gateway import forum_replies as fr

ME = "cbp-being"


def git(cwd, *args):
r = subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True,
env=dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t",
GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t"))
assert r.returncode == 0, r.stderr
return r.stdout.strip()


def today():
return time.strftime("%Y-%m-%d", time.gmtime())


def reply(name, body="The answer is on the forum.", to=ME):
return name, f"---\nfrom: pub-claude\nto: {to}\ndate: {today()}\nre: model trial\n---\n\n{body}\n"


def commit(clone, files, msg="post"):
(clone / "forum").mkdir(exist_ok=True)
for name, text in files:
(clone / "forum" / name).write_text(text)
git(clone, "add", "-A")
git(clone, "commit", "-q", "-m", msg)
git(clone, "push", "-q", "origin", "HEAD:main")


@pytest.fixture
def fleet(tmp_path):
"""origin (bare), the seat's checkout <root>/shared-context beside <root>/sage, and a peer clone."""
origin = tmp_path / "origin.git"
git(tmp_path, "init", "-q", "--bare", "-b", "main", str(origin))
peer = tmp_path / "peer"
git(tmp_path, "clone", "-q", str(origin), str(peer))
git(peer, "checkout", "-q", "-b", "main")
commit(peer, [("README.md", "forum\n")], "init")
root = tmp_path / "ai"
(root / "sage" / "home").mkdir(parents=True)
seat = root / "shared-context"
git(tmp_path, "clone", "-q", str(origin), str(seat))
return root / "sage", seat, peer, root / "sage" / "home"


def beat(home, workspace, **kw):
got = fr.for_beat(home, str(workspace), ME, **kw)
fr.acknowledge(home, got["offered"])
return got


def test_a_reply_pushed_after_the_last_pull_is_offered(fleet):
workspace, seat, peer, home = fleet
name, text = reply(f"pub-to-cbp-being-what-the-model-trial-is-{today()}.md")
commit(peer, [(name, text)])
assert not (seat / "forum" / name).exists(), "the seat's checkout is behind"
got = beat(home, workspace)
assert got["offered"] == [name]
assert got["upstream"]["fetched"] is True and got["upstream"]["offered"] == [name]
assert "The answer is on the forum." in got["block"]
assert f"origin/main:forum/{name} — not in this checkout yet" in got["block"]
assert beat(home, workspace)["offered"] == [], "delivered once, by name, like any reply"


def test_the_shared_checkout_is_never_moved(fleet):
workspace, seat, peer, home = fleet
(seat / "scratch.txt").write_text("a sibling's uncommitted work")
head = git(seat, "rev-parse", "HEAD")
commit(peer, [reply(f"hub-to-cbp-being-how-you-are-categorized-{today()}.md")])
assert beat(home, workspace)["offered"]
assert git(seat, "rev-parse", "HEAD") == head
assert git(seat, "status", "--porcelain") == "?? scratch.txt"
assert sorted(x.name for x in (seat / "forum").iterdir()) == ["README.md"]


def test_a_post_already_on_disk_is_read_from_disk_once(fleet):
workspace, seat, peer, home = fleet
name, text = reply(f"pub-re-cbp-being-{today()}.md")
commit(peer, [(name, text)])
git(seat, "pull", "-q", "--ff-only")
got = beat(home, workspace)
assert got["offered"] == [name] and got["upstream"]["unpulled"] == 0
assert f"shared-context/forum/{name}" in got["block"] and "not in this checkout" not in got["block"]


def test_a_reply_offered_from_upstream_is_not_offered_again_once_pulled(fleet):
workspace, seat, peer, home = fleet
name, text = reply(f"pub-re-cbp-being-{today()}.md")
commit(peer, [(name, text)])
assert beat(home, workspace)["offered"] == [name]
git(seat, "pull", "-q", "--ff-only")
assert beat(home, workspace)["offered"] == []


def test_posts_not_addressed_to_the_being_stay_unoffered(fleet):
workspace, seat, peer, home = fleet
commit(peer, [reply(f"pub-to-sprout-being-{today()}.md", to="sprout-being"),
reply(f"cbp-being-asks-pub-{today()}.md")])
got = beat(home, workspace)
assert got["offered"] == [] and got["upstream"]["unpulled"] == 2


def test_fetch_is_throttled_by_the_last_fetch(fleet):
workspace, seat, peer, home = fleet
assert fr.for_beat(home, str(workspace), ME)["upstream"]["fetched"] is True
commit(peer, [reply(f"pub-re-cbp-being-{today()}.md")])
got = fr.for_beat(home, str(workspace), ME)
assert got["upstream"]["fetched"] is False and got["offered"] == [], "fetched under FETCH_EVERY_S ago"
later = time.time() + fr.FETCH_EVERY_S + 1
got = fr.for_beat(home, str(workspace), ME, now=later)
assert got["upstream"]["fetched"] is True and len(got["offered"]) == 1


def test_fetch_can_be_turned_off_and_what_origin_holds_is_still_read(fleet, monkeypatch):
workspace, seat, peer, home = fleet
name, text = reply(f"pub-re-cbp-being-{today()}.md")
commit(peer, [(name, text)])
monkeypatch.setenv("SAGE_FORUM_FETCH", "0")
assert fr.for_beat(home, str(workspace), ME)["offered"] == []
git(seat, "fetch", "-q", "origin") # some session fetched; the tree is still behind
got = fr.for_beat(home, str(workspace), ME)
assert got["offered"] == [name] and got["upstream"]["fetched"] is False


def test_an_unreachable_origin_is_said_and_disk_replies_still_come(fleet):
workspace, seat, peer, home = fleet
name, text = reply(f"pub-re-cbp-being-{today()}.md")
(seat / "forum" / name).write_text(text)
git(seat, "remote", "set-url", "origin", str(seat.parent / "gone.git"))
got = fr.for_beat(home, str(workspace), ME, now=time.time() + fr.FETCH_EVERY_S + 1)
assert got["offered"] == [name]
assert got["upstream"]["fetched"] is False and got["upstream"]["why"].startswith("fetch failed")


def test_a_forum_that_is_not_a_git_checkout_runs_no_git(tmp_path, monkeypatch):
(tmp_path / "ai" / "sage" / "home").mkdir(parents=True)
(tmp_path / "ai" / "shared-context" / "forum").mkdir(parents=True)
monkeypatch.setattr(fr, "_git", lambda *a, **k: pytest.fail("git ran outside a checkout"))
got = fr.for_beat(tmp_path / "ai" / "sage" / "home", str(tmp_path / "ai" / "sage"), ME)
assert got["offered"] == [] and got["upstream"]["why"] == "not a git checkout"
Loading