diff --git a/changes/fix-conversations-forum-topics.md b/changes/fix-conversations-forum-topics.md new file mode 100644 index 000000000..4f42acfea --- /dev/null +++ b/changes/fix-conversations-forum-topics.md @@ -0,0 +1,10 @@ +### Fixed + +- **A conversation never mixes the topics of a forum group.** A message's parent is a message of its own topic or + thread, or the thread's first message when that has text, as a WhatsApp community post its comments answer. In a + Telegram forum every message of a topic replies to the topic's empty first message: taken as a reply, it chained a + whole topic, and through those first messages' sender every topic, into one conversation, and its chunks mixed + topics. A messenger reply into another topic, an agent's answer naming one, and the rules never link across. The + linking-rules version is 6, so every chat built before reads as outdated and is rebuilt; a chunk whose text is + unchanged keeps its vector. `conversations batches next` now asks the agent about a message whose only reply is + one the build does not link. diff --git a/src/conversations/link.test.ts b/src/conversations/link.test.ts index 1d5c433ea..554c214b0 100644 --- a/src/conversations/link.test.ts +++ b/src/conversations/link.test.ts @@ -109,8 +109,8 @@ describe("linkMessages", () => { expect(parents.get("3")).toBe("2") }) - it("never links across forum threads by a rule, but keeps an explicit reply", () => { - const { parents } = linkMessages( + it("never links across forum threads, not by a rule and not by an explicit reply", () => { + const { parents, links } = linkMessages( [ said("1", "a", "visa question", { threadId: "t1" }), said("2", "a", "also", { threadId: "t2" }), @@ -121,7 +121,46 @@ describe("linkMessages", () => { ) expect(parents.get("2")).toBeNull() expect(parents.get("3")).toBe("2") - expect(parents.get("4")).toBe("1") + expect(parents.get("4")).toBe("3") + expect(links).not.toContainEqual(expect.objectContaining({ messageId: "4", parentId: "1" })) + }) + + it("**keeps each forum topic apart**: a reply to the topic's first message is no link, a reply inside it is", () => { + const { conversations, links } = linkMessages([ + said("10", "a", ""), + said("20", "a", ""), + said("11", "b", "who has the hall key?", { threadId: "10", replyToId: "10" }), + said("21", "c", "the boat leaves at nine", { threadId: "20", replyToId: "20" }), + said("12", "d", "I do", { threadId: "10", replyToId: "11" }), + said("22", "e", "noted", { threadId: "20", replyToId: "20" }), + ]) + + expect(conversations).toEqual([["10", "20"], ["11", "12"], ["21"], ["22"]]) + expect(links.filter(({ parentId }) => parentId === "10" || parentId === "20")).toEqual([ + expect.objectContaining({ messageId: "20", kind: "same_sender" }), + ]) + }) + + it("keeps a comment linked to the post it comments on, a thread whose first message has text", () => { + const { conversations } = linkMessages([ + said("p1", "a", "Garden cleanup on Saturday, bring gloves"), + said("p2", "d", "The pool opens in June"), + said("c1", "b", "I can bring two rakes", { threadId: "p1", replyToId: "p1" }), + said("c2", "c", "what time?", { threadId: "p2", replyToId: "p2" }), + ]) + + expect(conversations).toEqual([ + ["p1", "c1"], + ["p2", "c2"], + ]) + }) + + it("leaves the choice to the rules when the agent names a message in another topic", () => { + const { parents } = linkMessages( + [said("1", "a", "lunch?", { threadId: "t1" }), said("2", "b", "rain later", { threadId: "t2" })], + { answers: new Map([["2", "1"]]) }, + ) + expect(parents.get("2")).toBeNull() }) it("starts a conversation for a reply to a message it does not hold", () => { diff --git a/src/conversations/link.ts b/src/conversations/link.ts index 3390a2e0c..4dd543368 100644 --- a/src/conversations/link.ts +++ b/src/conversations/link.ts @@ -1,7 +1,10 @@ import type { Id, Message } from "../domain/models.js" -/** 5: a message longer than a chunk is split into overlapping pieces, so builds before it are outdated. */ -export const RULES_VERSION = 5 +/** + * 5: a message longer than a chunk is split into overlapping pieces. 6: a message's parent is always in its own + * topic or thread, so a forum's topics are no longer one conversation. Builds before the current version are outdated. + */ +export const RULES_VERSION = 6 /** How far back a rule looks, in messages: 97% of reply parents sat within 50 in a measured group. */ export const LOOK_BACK = 50 @@ -55,24 +58,26 @@ export const linkMessages = ( const links: Link[] = [] const parents = new Map() const conversationOf = new Map() + const held = new Map() const conversations: Id[][] = [] const window: LinkInput[] = [] for (const message of messages) { check?.() const found = [ - ...replyLink(message, conversationOf), + ...replyLink(message, held), ...mentionLinks(message, window, handles), ...sameSenderLink(message, window), ] links.push(...found) - const parent = choose(found, answers, message.id, conversationOf) + const parent = choose(found, answers, message, held) parents.set(message.id, parent) const conversation = parent === null ? conversations.length : (conversationOf.get(parent) as number) if (conversation === conversations.length) conversations.push([]) conversations[conversation]?.push(message.id) conversationOf.set(message.id, conversation) + held.set(message.id, { threadId: message.threadId, text: message.text.trim() !== "" }) window.push(message) if (window.length > LOOK_BACK) window.shift() @@ -80,11 +85,28 @@ export const linkMessages = ( return { links, parents, conversations } } -const replyLink = (message: LinkInput, held: ReadonlyMap): Link[] => - message.replyToId !== undefined && held.has(message.replyToId) +interface Held { + threadId: Id | undefined + text: boolean +} + +const replyLink = (message: LinkInput, held: ReadonlyMap): Link[] => + message.replyToId !== undefined && inThread(held, message.replyToId, message) ? [link(message, message.replyToId, "provider", "reply", 1, "reply")] : [] +/** + * Whether `id` may be the message's parent: a message of the same topic or thread, or the thread's own first + * message when it says something, as a WhatsApp community post its comments answer. A Telegram forum topic starts + * with an empty service message every message of the topic replies to: linked, it chained the whole topic, and + * through those first messages' sender every topic, into one conversation. + */ +const inThread = (held: ReadonlyMap, id: Id, message: LinkInput) => { + const parent = held.get(id) + if (!parent) return false + return parent.threadId === message.threadId || (id === message.threadId && parent.text) +} + const mentionLinks = (message: LinkInput, window: LinkInput[], handles: ReadonlyMap): Link[] => { const links: Link[] = [] const people = new Set([...mentioned(message.text, handles)].flatMap((handle) => handles.get(handle) ?? [])) @@ -123,15 +145,16 @@ const mentioned = (text: string, handles: ReadonlyMap): Set const choose = ( links: Link[], answers: ReadonlyMap, - id: Id, - held: ReadonlyMap, + message: LinkInput, + held: ReadonlyMap, ): Id | null => { const provider = links.find((one) => one.source === "provider") if (provider) return provider.parentId - // An answer naming a message this chat no longer holds, or a later one, leaves the choice to the rules. - const answer = answers.get(id) + // An answer naming a message this chat no longer holds, a later one or one in another topic leaves the choice + // to the rules. + const answer = answers.get(message.id) if (answer === null) return null - if (answer !== undefined && held.has(answer)) return answer + if (answer !== undefined && inThread(held, answer, message)) return answer let best: Link | undefined for (const one of links) if (!best || one.confidence > best.confidence) best = one return best?.parentId ?? null diff --git a/src/services/thread-context.test.ts b/src/services/thread-context.test.ts index f30bdb3c7..50d09571d 100644 --- a/src/services/thread-context.test.ts +++ b/src/services/thread-context.test.ts @@ -252,11 +252,11 @@ describe("graph context around a hit", () => { expect(found.stopped).toContain("time") }) - it("does not cross native topics", async () => { + it("does not cross native topics: a reply into another topic is no link", async () => { const one = await setup(true, [message(1), { ...message(2, 1), threadId: "topic" }]) const found = await one.services.messages.thread("7", "2") expect(ids(found)).toEqual(["2"]) - expect(found.stopped).toContain("thread") + expect(found.links).toEqual([]) }) it("keeps other accounts and chats isolated even with the same message ids", async () => { diff --git a/src/store/batches.test.ts b/src/store/batches.test.ts index 8c87cb5fe..d2886dede 100644 --- a/src/store/batches.test.ts +++ b/src/store/batches.test.ts @@ -81,6 +81,31 @@ describe("batches for the user's agent", () => { await store.close() }) + it("asks about a forum message whose only reply is to its topic's empty first message, not a post's comment", async () => { + const path = join(mkdtempSync(join(tmpdir(), "batches-")), "messages.db") + const store = await openStore({ path }) + await store.chats.saveChats(OWNER, [ + { id: "-1", title: "Group", kind: "group", unreadCount: 0, lastMessageAt: null, participantsCount: null }, + ]) + await store.messages.saveMessages( + OWNER, + "-1", + [ + { ...message(1), text: "" }, + { ...message(2, "1"), threadId: "1" }, + { ...message(3, "2"), threadId: "1" }, + message(4), + { ...message(5, "4"), threadId: "4" }, + ], + { via: "history" }, + ) + + const batch = await store.conversations.nextBatch(OWNER, "-1", { size: 50 }) + + expect(ids(batch?.messages ?? [], true)).toEqual([1, 2, 4]) + await store.close() + }) + it("asks again about a message whose answer went stale, and about nothing once all are answered", async () => { const { store, path } = await chatOf(3) await answered(path, 1) diff --git a/src/store/sqlite/batches.ts b/src/store/sqlite/batches.ts index 283cde0b7..0b9c1c230 100644 --- a/src/store/sqlite/batches.ts +++ b/src/store/sqlite/batches.ts @@ -11,10 +11,13 @@ export const BATCH_CONTEXT = 50 /** * Phase 4 plan A2: a live message the agent is asked about — no reply the messenger itself records to a - * message held here, and no current agent answer. A stale answer makes it need the agent again. + * message held here that the build links (`inThread` in conversations/link.ts), and no current agent answer. + * A stale answer makes it need the agent again. */ const replied = (m: SQL) => sql`EXISTS (SELECT 1 FROM messages p WHERE p.chat_id = ${m}.chat_id - AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL)` + AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL + AND (p.thread_external_id IS ${m}.thread_external_id + OR (p.external_id = ${m}.thread_external_id AND trim(p.text) <> '')))` const needsAgent = (m: SQL) => sql`( NOT ${replied(m)}