From 80c941f20762e92241934013f5a0831d07405c7e Mon Sep 17 00:00:00 2001 From: leemour Date: Tue, 17 Nov 2026 22:50:05 +0100 Subject: [PATCH 1/2] fix(conversations): a conversation never mixes forum topics A message's parent is now always a message in its own topic or thread. A messenger reply into another topic, an agent's answer naming one, and the rules (as before) never link across. In a Telegram forum every message of a topic replies to the topic's first message, which carries no topic itself. Taken as a reply, it chained a whole topic into one conversation, and the first messages, written by one person minutes apart, were joined by the same-sender rule, so every topic of the forum became one conversation whose chunks mixed topics. RULES_VERSION is 6, so chats built before read as outdated and are rebuilt; a chunk whose text did not change keeps its vector (embeddings are keyed by the chunk's hash). The agent batches ask about a message whose only reply points into another topic, as the build no longer counts it as replied. Co-Authored-By: Claude Opus 5.5 --- changes/fix-conversations-forum-topics.md | 9 ++++++ src/conversations/link.test.ts | 31 +++++++++++++++++-- src/conversations/link.ts | 36 ++++++++++++++++------- src/services/thread-context.test.ts | 4 +-- src/store/batches.test.ts | 19 ++++++++++++ src/store/sqlite/batches.ts | 5 ++-- 6 files changed, 86 insertions(+), 18 deletions(-) create mode 100644 changes/fix-conversations-forum-topics.md diff --git a/changes/fix-conversations-forum-topics.md b/changes/fix-conversations-forum-topics.md new file mode 100644 index 000000000..79044d289 --- /dev/null +++ b/changes/fix-conversations-forum-topics.md @@ -0,0 +1,9 @@ +### Fixed + +- **A conversation never mixes the topics of a forum group.** A message's parent is always a message in its own topic + or thread: a messenger reply into another topic, an agent's answer naming one, and the rules never link across. In a + Telegram forum every message of a topic replies to the topic's first message, which has no topic itself; taken as a + reply, it chained a whole topic, and through the first messages' sender every topic, into one conversation, and its + chunks mixed topics. The linking-rules version is 6, so a chat built before reads as outdated and is rebuilt; a + chunk whose text is unchanged keeps its vector. `conversations batches next` now asks the agent about a message + whose only reply is into another topic. diff --git a/src/conversations/link.test.ts b/src/conversations/link.test.ts index 1d5c433ea..bc6c52712 100644 --- a/src/conversations/link.test.ts +++ b/src/conversations/link.test.ts @@ -109,8 +109,8 @@ describe("linkMessages", () => { expect(parents.get("3")).toBe("2") }) - it("never links across forum threads by a rule, but keeps an explicit reply", () => { - const { parents } = linkMessages( + it("never links across forum threads, not by a rule and not by an explicit reply", () => { + const { parents, links } = linkMessages( [ said("1", "a", "visa question", { threadId: "t1" }), said("2", "a", "also", { threadId: "t2" }), @@ -121,7 +121,32 @@ describe("linkMessages", () => { ) expect(parents.get("2")).toBeNull() expect(parents.get("3")).toBe("2") - expect(parents.get("4")).toBe("1") + expect(parents.get("4")).toBe("3") + expect(links).not.toContainEqual(expect.objectContaining({ messageId: "4", parentId: "1" })) + }) + + it("**keeps each forum topic apart**: a reply to the topic's first message is no link, a reply inside it is", () => { + const { conversations, links } = linkMessages([ + said("10", "a", ""), + said("20", "a", ""), + said("11", "b", "who has the hall key?", { threadId: "10", replyToId: "10" }), + said("21", "c", "the boat leaves at nine", { threadId: "20", replyToId: "20" }), + said("12", "d", "I do", { threadId: "10", replyToId: "11" }), + said("22", "e", "noted", { threadId: "20", replyToId: "20" }), + ]) + + expect(conversations).toEqual([["10", "20"], ["11", "12"], ["21"], ["22"]]) + expect(links.filter(({ parentId }) => parentId === "10" || parentId === "20")).toEqual([ + expect.objectContaining({ messageId: "20", kind: "same_sender" }), + ]) + }) + + it("leaves the choice to the rules when the agent names a message in another topic", () => { + const { parents } = linkMessages( + [said("1", "a", "lunch?", { threadId: "t1" }), said("2", "b", "rain later", { threadId: "t2" })], + { answers: new Map([["2", "1"]]) }, + ) + expect(parents.get("2")).toBeNull() }) it("starts a conversation for a reply to a message it does not hold", () => { diff --git a/src/conversations/link.ts b/src/conversations/link.ts index 3390a2e0c..e5cc972f0 100644 --- a/src/conversations/link.ts +++ b/src/conversations/link.ts @@ -1,7 +1,10 @@ import type { Id, Message } from "../domain/models.js" -/** 5: a message longer than a chunk is split into overlapping pieces, so builds before it are outdated. */ -export const RULES_VERSION = 5 +/** + * 5: a message longer than a chunk is split into overlapping pieces. 6: a message's parent is always in its own + * topic or thread, so a forum's topics are no longer one conversation. Builds before the current version are outdated. + */ +export const RULES_VERSION = 6 /** How far back a rule looks, in messages: 97% of reply parents sat within 50 in a measured group. */ export const LOOK_BACK = 50 @@ -55,24 +58,26 @@ export const linkMessages = ( const links: Link[] = [] const parents = new Map() const conversationOf = new Map() + const threadOf = new Map() const conversations: Id[][] = [] const window: LinkInput[] = [] for (const message of messages) { check?.() const found = [ - ...replyLink(message, conversationOf), + ...replyLink(message, threadOf), ...mentionLinks(message, window, handles), ...sameSenderLink(message, window), ] links.push(...found) - const parent = choose(found, answers, message.id, conversationOf) + const parent = choose(found, answers, message, threadOf) parents.set(message.id, parent) const conversation = parent === null ? conversations.length : (conversationOf.get(parent) as number) if (conversation === conversations.length) conversations.push([]) conversations[conversation]?.push(message.id) conversationOf.set(message.id, conversation) + threadOf.set(message.id, message.threadId) window.push(message) if (window.length > LOOK_BACK) window.shift() @@ -80,11 +85,19 @@ export const linkMessages = ( return { links, parents, conversations } } -const replyLink = (message: LinkInput, held: ReadonlyMap): Link[] => - message.replyToId !== undefined && held.has(message.replyToId) +/** + * A reply into another topic or thread is no link. In a Telegram forum every message of a topic replies to the + * topic's first message, which has no topic of its own: taken as a reply, it chained the whole topic, and through + * that first message's sender, every topic, into one conversation. + */ +const replyLink = (message: LinkInput, threads: ReadonlyMap): Link[] => + message.replyToId !== undefined && inThread(threads, message.replyToId, message) ? [link(message, message.replyToId, "provider", "reply", 1, "reply")] : [] +const inThread = (threads: ReadonlyMap, id: Id, message: LinkInput) => + threads.has(id) && threads.get(id) === message.threadId + const mentionLinks = (message: LinkInput, window: LinkInput[], handles: ReadonlyMap): Link[] => { const links: Link[] = [] const people = new Set([...mentioned(message.text, handles)].flatMap((handle) => handles.get(handle) ?? [])) @@ -123,15 +136,16 @@ const mentioned = (text: string, handles: ReadonlyMap): Set const choose = ( links: Link[], answers: ReadonlyMap, - id: Id, - held: ReadonlyMap, + message: LinkInput, + threads: ReadonlyMap, ): Id | null => { const provider = links.find((one) => one.source === "provider") if (provider) return provider.parentId - // An answer naming a message this chat no longer holds, or a later one, leaves the choice to the rules. - const answer = answers.get(id) + // An answer naming a message this chat no longer holds, a later one or one in another topic leaves the choice + // to the rules. + const answer = answers.get(message.id) if (answer === null) return null - if (answer !== undefined && held.has(answer)) return answer + if (answer !== undefined && inThread(threads, answer, message)) return answer let best: Link | undefined for (const one of links) if (!best || one.confidence > best.confidence) best = one return best?.parentId ?? null diff --git a/src/services/thread-context.test.ts b/src/services/thread-context.test.ts index f30bdb3c7..50d09571d 100644 --- a/src/services/thread-context.test.ts +++ b/src/services/thread-context.test.ts @@ -252,11 +252,11 @@ describe("graph context around a hit", () => { expect(found.stopped).toContain("time") }) - it("does not cross native topics", async () => { + it("does not cross native topics: a reply into another topic is no link", async () => { const one = await setup(true, [message(1), { ...message(2, 1), threadId: "topic" }]) const found = await one.services.messages.thread("7", "2") expect(ids(found)).toEqual(["2"]) - expect(found.stopped).toContain("thread") + expect(found.links).toEqual([]) }) it("keeps other accounts and chats isolated even with the same message ids", async () => { diff --git a/src/store/batches.test.ts b/src/store/batches.test.ts index 8c87cb5fe..2868f0016 100644 --- a/src/store/batches.test.ts +++ b/src/store/batches.test.ts @@ -81,6 +81,25 @@ describe("batches for the user's agent", () => { await store.close() }) + it("asks about a forum message whose only reply is to its topic's first message", async () => { + const path = join(mkdtempSync(join(tmpdir(), "batches-")), "messages.db") + const store = await openStore({ path }) + await store.chats.saveChats(OWNER, [ + { id: "-1", title: "Group", kind: "group", unreadCount: 0, lastMessageAt: null, participantsCount: null }, + ]) + await store.messages.saveMessages( + OWNER, + "-1", + [message(1), { ...message(2, "1"), threadId: "1" }, { ...message(3, "2"), threadId: "1" }], + { via: "history" }, + ) + + const batch = await store.conversations.nextBatch(OWNER, "-1", { size: 50 }) + + expect(ids(batch?.messages ?? [], true)).toEqual([1, 2]) + await store.close() + }) + it("asks again about a message whose answer went stale, and about nothing once all are answered", async () => { const { store, path } = await chatOf(3) await answered(path, 1) diff --git a/src/store/sqlite/batches.ts b/src/store/sqlite/batches.ts index 283cde0b7..c825faae8 100644 --- a/src/store/sqlite/batches.ts +++ b/src/store/sqlite/batches.ts @@ -11,10 +11,11 @@ export const BATCH_CONTEXT = 50 /** * Phase 4 plan A2: a live message the agent is asked about — no reply the messenger itself records to a - * message held here, and no current agent answer. A stale answer makes it need the agent again. + * message held here in the same topic, and no current agent answer. A stale answer makes it need the agent again. */ const replied = (m: SQL) => sql`EXISTS (SELECT 1 FROM messages p WHERE p.chat_id = ${m}.chat_id - AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL)` + AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL + AND p.thread_external_id IS ${m}.thread_external_id)` const needsAgent = (m: SQL) => sql`( NOT ${replied(m)} From e329f36878dde636b4cd78154994a2afc0f77577 Mon Sep 17 00:00:00 2001 From: leemour Date: Wed, 18 Nov 2026 00:49:54 +0100 Subject: [PATCH 2/2] fix(conversations): a comment stays linked to a thread's first message with text A WhatsApp community comment carries the post as its thread and replies to it, while the post itself has no thread. Requiring the same thread cut every comment off its post. A reply to the thread's own first message is now a link when that message has text; a Telegram forum topic's first message is an empty service message, so forum topics still stay apart (checked on a test store: none of the 30 stored topic roots has text). The agent batches count a reply as recorded by the same rule. Co-Authored-By: Claude Opus 5.5 --- changes/fix-conversations-forum-topics.md | 15 +++++---- src/conversations/link.test.ts | 14 ++++++++ src/conversations/link.ts | 39 ++++++++++++++--------- src/store/batches.test.ts | 12 +++++-- src/store/sqlite/batches.ts | 6 ++-- 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/changes/fix-conversations-forum-topics.md b/changes/fix-conversations-forum-topics.md index 79044d289..4f42acfea 100644 --- a/changes/fix-conversations-forum-topics.md +++ b/changes/fix-conversations-forum-topics.md @@ -1,9 +1,10 @@ ### Fixed -- **A conversation never mixes the topics of a forum group.** A message's parent is always a message in its own topic - or thread: a messenger reply into another topic, an agent's answer naming one, and the rules never link across. In a - Telegram forum every message of a topic replies to the topic's first message, which has no topic itself; taken as a - reply, it chained a whole topic, and through the first messages' sender every topic, into one conversation, and its - chunks mixed topics. The linking-rules version is 6, so a chat built before reads as outdated and is rebuilt; a - chunk whose text is unchanged keeps its vector. `conversations batches next` now asks the agent about a message - whose only reply is into another topic. +- **A conversation never mixes the topics of a forum group.** A message's parent is a message of its own topic or + thread, or the thread's first message when that has text, as a WhatsApp community post its comments answer. In a + Telegram forum every message of a topic replies to the topic's empty first message: taken as a reply, it chained a + whole topic, and through those first messages' sender every topic, into one conversation, and its chunks mixed + topics. A messenger reply into another topic, an agent's answer naming one, and the rules never link across. The + linking-rules version is 6, so every chat built before reads as outdated and is rebuilt; a chunk whose text is + unchanged keeps its vector. `conversations batches next` now asks the agent about a message whose only reply is + one the build does not link. diff --git a/src/conversations/link.test.ts b/src/conversations/link.test.ts index bc6c52712..554c214b0 100644 --- a/src/conversations/link.test.ts +++ b/src/conversations/link.test.ts @@ -141,6 +141,20 @@ describe("linkMessages", () => { ]) }) + it("keeps a comment linked to the post it comments on, a thread whose first message has text", () => { + const { conversations } = linkMessages([ + said("p1", "a", "Garden cleanup on Saturday, bring gloves"), + said("p2", "d", "The pool opens in June"), + said("c1", "b", "I can bring two rakes", { threadId: "p1", replyToId: "p1" }), + said("c2", "c", "what time?", { threadId: "p2", replyToId: "p2" }), + ]) + + expect(conversations).toEqual([ + ["p1", "c1"], + ["p2", "c2"], + ]) + }) + it("leaves the choice to the rules when the agent names a message in another topic", () => { const { parents } = linkMessages( [said("1", "a", "lunch?", { threadId: "t1" }), said("2", "b", "rain later", { threadId: "t2" })], diff --git a/src/conversations/link.ts b/src/conversations/link.ts index e5cc972f0..4dd543368 100644 --- a/src/conversations/link.ts +++ b/src/conversations/link.ts @@ -58,26 +58,26 @@ export const linkMessages = ( const links: Link[] = [] const parents = new Map() const conversationOf = new Map() - const threadOf = new Map() + const held = new Map() const conversations: Id[][] = [] const window: LinkInput[] = [] for (const message of messages) { check?.() const found = [ - ...replyLink(message, threadOf), + ...replyLink(message, held), ...mentionLinks(message, window, handles), ...sameSenderLink(message, window), ] links.push(...found) - const parent = choose(found, answers, message, threadOf) + const parent = choose(found, answers, message, held) parents.set(message.id, parent) const conversation = parent === null ? conversations.length : (conversationOf.get(parent) as number) if (conversation === conversations.length) conversations.push([]) conversations[conversation]?.push(message.id) conversationOf.set(message.id, conversation) - threadOf.set(message.id, message.threadId) + held.set(message.id, { threadId: message.threadId, text: message.text.trim() !== "" }) window.push(message) if (window.length > LOOK_BACK) window.shift() @@ -85,18 +85,27 @@ export const linkMessages = ( return { links, parents, conversations } } -/** - * A reply into another topic or thread is no link. In a Telegram forum every message of a topic replies to the - * topic's first message, which has no topic of its own: taken as a reply, it chained the whole topic, and through - * that first message's sender, every topic, into one conversation. - */ -const replyLink = (message: LinkInput, threads: ReadonlyMap): Link[] => - message.replyToId !== undefined && inThread(threads, message.replyToId, message) +interface Held { + threadId: Id | undefined + text: boolean +} + +const replyLink = (message: LinkInput, held: ReadonlyMap): Link[] => + message.replyToId !== undefined && inThread(held, message.replyToId, message) ? [link(message, message.replyToId, "provider", "reply", 1, "reply")] : [] -const inThread = (threads: ReadonlyMap, id: Id, message: LinkInput) => - threads.has(id) && threads.get(id) === message.threadId +/** + * Whether `id` may be the message's parent: a message of the same topic or thread, or the thread's own first + * message when it says something, as a WhatsApp community post its comments answer. A Telegram forum topic starts + * with an empty service message every message of the topic replies to: linked, it chained the whole topic, and + * through those first messages' sender every topic, into one conversation. + */ +const inThread = (held: ReadonlyMap, id: Id, message: LinkInput) => { + const parent = held.get(id) + if (!parent) return false + return parent.threadId === message.threadId || (id === message.threadId && parent.text) +} const mentionLinks = (message: LinkInput, window: LinkInput[], handles: ReadonlyMap): Link[] => { const links: Link[] = [] @@ -137,7 +146,7 @@ const choose = ( links: Link[], answers: ReadonlyMap, message: LinkInput, - threads: ReadonlyMap, + held: ReadonlyMap, ): Id | null => { const provider = links.find((one) => one.source === "provider") if (provider) return provider.parentId @@ -145,7 +154,7 @@ const choose = ( // to the rules. const answer = answers.get(message.id) if (answer === null) return null - if (answer !== undefined && inThread(threads, answer, message)) return answer + if (answer !== undefined && inThread(held, answer, message)) return answer let best: Link | undefined for (const one of links) if (!best || one.confidence > best.confidence) best = one return best?.parentId ?? null diff --git a/src/store/batches.test.ts b/src/store/batches.test.ts index 2868f0016..d2886dede 100644 --- a/src/store/batches.test.ts +++ b/src/store/batches.test.ts @@ -81,7 +81,7 @@ describe("batches for the user's agent", () => { await store.close() }) - it("asks about a forum message whose only reply is to its topic's first message", async () => { + it("asks about a forum message whose only reply is to its topic's empty first message, not a post's comment", async () => { const path = join(mkdtempSync(join(tmpdir(), "batches-")), "messages.db") const store = await openStore({ path }) await store.chats.saveChats(OWNER, [ @@ -90,13 +90,19 @@ describe("batches for the user's agent", () => { await store.messages.saveMessages( OWNER, "-1", - [message(1), { ...message(2, "1"), threadId: "1" }, { ...message(3, "2"), threadId: "1" }], + [ + { ...message(1), text: "" }, + { ...message(2, "1"), threadId: "1" }, + { ...message(3, "2"), threadId: "1" }, + message(4), + { ...message(5, "4"), threadId: "4" }, + ], { via: "history" }, ) const batch = await store.conversations.nextBatch(OWNER, "-1", { size: 50 }) - expect(ids(batch?.messages ?? [], true)).toEqual([1, 2]) + expect(ids(batch?.messages ?? [], true)).toEqual([1, 2, 4]) await store.close() }) diff --git a/src/store/sqlite/batches.ts b/src/store/sqlite/batches.ts index c825faae8..0b9c1c230 100644 --- a/src/store/sqlite/batches.ts +++ b/src/store/sqlite/batches.ts @@ -11,11 +11,13 @@ export const BATCH_CONTEXT = 50 /** * Phase 4 plan A2: a live message the agent is asked about — no reply the messenger itself records to a - * message held here in the same topic, and no current agent answer. A stale answer makes it need the agent again. + * message held here that the build links (`inThread` in conversations/link.ts), and no current agent answer. + * A stale answer makes it need the agent again. */ const replied = (m: SQL) => sql`EXISTS (SELECT 1 FROM messages p WHERE p.chat_id = ${m}.chat_id AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL - AND p.thread_external_id IS ${m}.thread_external_id)` + AND (p.thread_external_id IS ${m}.thread_external_id + OR (p.external_id = ${m}.thread_external_id AND trim(p.text) <> '')))` const needsAgent = (m: SQL) => sql`( NOT ${replied(m)}