Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions changes/fix-conversations-forum-topics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
### Fixed

- **A conversation never mixes the topics of a forum group.** A message's parent is a message of its own topic or
thread, or the thread's first message when that has text, as a WhatsApp community post its comments answer. In a
Telegram forum every message of a topic replies to the topic's empty first message: taken as a reply, it chained a
whole topic, and through those first messages' sender every topic, into one conversation, and its chunks mixed
topics. A messenger reply into another topic, an agent's answer naming one, and the rules never link across. The
linking-rules version is 6, so every chat built before reads as outdated and is rebuilt; a chunk whose text is
unchanged keeps its vector. `conversations batches next` now asks the agent about a message whose only reply is
one the build does not link.
45 changes: 42 additions & 3 deletions src/conversations/link.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ describe("linkMessages", () => {
expect(parents.get("3")).toBe("2")
})

it("never links across forum threads by a rule, but keeps an explicit reply", () => {
const { parents } = linkMessages(
it("never links across forum threads, not by a rule and not by an explicit reply", () => {
const { parents, links } = linkMessages(
[
said("1", "a", "visa question", { threadId: "t1" }),
said("2", "a", "also", { threadId: "t2" }),
Expand All @@ -121,7 +121,46 @@ describe("linkMessages", () => {
)
expect(parents.get("2")).toBeNull()
expect(parents.get("3")).toBe("2")
expect(parents.get("4")).toBe("1")
expect(parents.get("4")).toBe("3")
expect(links).not.toContainEqual(expect.objectContaining({ messageId: "4", parentId: "1" }))
})

it("**keeps each forum topic apart**: a reply to the topic's first message is no link, a reply inside it is", () => {
const { conversations, links } = linkMessages([
said("10", "a", ""),
said("20", "a", ""),
said("11", "b", "who has the hall key?", { threadId: "10", replyToId: "10" }),
said("21", "c", "the boat leaves at nine", { threadId: "20", replyToId: "20" }),
said("12", "d", "I do", { threadId: "10", replyToId: "11" }),
said("22", "e", "noted", { threadId: "20", replyToId: "20" }),
])

expect(conversations).toEqual([["10", "20"], ["11", "12"], ["21"], ["22"]])
expect(links.filter(({ parentId }) => parentId === "10" || parentId === "20")).toEqual([
expect.objectContaining({ messageId: "20", kind: "same_sender" }),
])
})

it("keeps a comment linked to the post it comments on, a thread whose first message has text", () => {
const { conversations } = linkMessages([
said("p1", "a", "Garden cleanup on Saturday, bring gloves"),
said("p2", "d", "The pool opens in June"),
said("c1", "b", "I can bring two rakes", { threadId: "p1", replyToId: "p1" }),
said("c2", "c", "what time?", { threadId: "p2", replyToId: "p2" }),
])

expect(conversations).toEqual([
["p1", "c1"],
["p2", "c2"],
])
})

it("leaves the choice to the rules when the agent names a message in another topic", () => {
const { parents } = linkMessages(
[said("1", "a", "lunch?", { threadId: "t1" }), said("2", "b", "rain later", { threadId: "t2" })],
{ answers: new Map([["2", "1"]]) },
)
expect(parents.get("2")).toBeNull()
})

it("starts a conversation for a reply to a message it does not hold", () => {
Expand Down
45 changes: 34 additions & 11 deletions src/conversations/link.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import type { Id, Message } from "../domain/models.js"

/** 5: a message longer than a chunk is split into overlapping pieces, so builds before it are outdated. */
export const RULES_VERSION = 5
/**
* 5: a message longer than a chunk is split into overlapping pieces. 6: a message's parent is always in its own
* topic or thread, so a forum's topics are no longer one conversation. Builds before the current version are outdated.
*/
export const RULES_VERSION = 6

/** How far back a rule looks, in messages: 97% of reply parents sat within 50 in a measured group. */
export const LOOK_BACK = 50
Expand Down Expand Up @@ -55,36 +58,55 @@ export const linkMessages = (
const links: Link[] = []
const parents = new Map<Id, Id | null>()
const conversationOf = new Map<Id, number>()
const held = new Map<Id, Held>()
const conversations: Id[][] = []
const window: LinkInput[] = []

for (const message of messages) {
check?.()
const found = [
...replyLink(message, conversationOf),
...replyLink(message, held),
...mentionLinks(message, window, handles),
...sameSenderLink(message, window),
]
links.push(...found)
const parent = choose(found, answers, message.id, conversationOf)
const parent = choose(found, answers, message, held)
parents.set(message.id, parent)

const conversation = parent === null ? conversations.length : (conversationOf.get(parent) as number)
if (conversation === conversations.length) conversations.push([])
conversations[conversation]?.push(message.id)
conversationOf.set(message.id, conversation)
held.set(message.id, { threadId: message.threadId, text: message.text.trim() !== "" })

window.push(message)
if (window.length > LOOK_BACK) window.shift()
}
return { links, parents, conversations }
}

const replyLink = (message: LinkInput, held: ReadonlyMap<Id, number>): Link[] =>
message.replyToId !== undefined && held.has(message.replyToId)
interface Held {
threadId: Id | undefined
text: boolean
}

const replyLink = (message: LinkInput, held: ReadonlyMap<Id, Held>): Link[] =>
message.replyToId !== undefined && inThread(held, message.replyToId, message)
? [link(message, message.replyToId, "provider", "reply", 1, "reply")]
: []

/**
* Whether `id` may be the message's parent: a message of the same topic or thread, or the thread's own first
* message when it says something, as a WhatsApp community post its comments answer. A Telegram forum topic starts
* with an empty service message every message of the topic replies to: linked, it chained the whole topic, and
* through those first messages' sender every topic, into one conversation.
*/
const inThread = (held: ReadonlyMap<Id, Held>, id: Id, message: LinkInput) => {
const parent = held.get(id)
if (!parent) return false
return parent.threadId === message.threadId || (id === message.threadId && parent.text)
}

const mentionLinks = (message: LinkInput, window: LinkInput[], handles: ReadonlyMap<string, Id>): Link[] => {
const links: Link[] = []
const people = new Set([...mentioned(message.text, handles)].flatMap((handle) => handles.get(handle) ?? []))
Expand Down Expand Up @@ -123,15 +145,16 @@ const mentioned = (text: string, handles: ReadonlyMap<string, Id>): Set<string>
const choose = (
links: Link[],
answers: ReadonlyMap<Id, Id | null>,
id: Id,
held: ReadonlyMap<Id, number>,
message: LinkInput,
held: ReadonlyMap<Id, Held>,
): Id | null => {
const provider = links.find((one) => one.source === "provider")
if (provider) return provider.parentId
// An answer naming a message this chat no longer holds, or a later one, leaves the choice to the rules.
const answer = answers.get(id)
// An answer naming a message this chat no longer holds, a later one or one in another topic leaves the choice
// to the rules.
const answer = answers.get(message.id)
if (answer === null) return null
if (answer !== undefined && held.has(answer)) return answer
if (answer !== undefined && inThread(held, answer, message)) return answer
let best: Link | undefined
for (const one of links) if (!best || one.confidence > best.confidence) best = one
return best?.parentId ?? null
Expand Down
4 changes: 2 additions & 2 deletions src/services/thread-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -252,11 +252,11 @@ describe("graph context around a hit", () => {
expect(found.stopped).toContain("time")
})

it("does not cross native topics", async () => {
it("does not cross native topics: a reply into another topic is no link", async () => {
const one = await setup(true, [message(1), { ...message(2, 1), threadId: "topic" }])
const found = await one.services.messages.thread("7", "2")
expect(ids(found)).toEqual(["2"])
expect(found.stopped).toContain("thread")
expect(found.links).toEqual([])
})

it("keeps other accounts and chats isolated even with the same message ids", async () => {
Expand Down
25 changes: 25 additions & 0 deletions src/store/batches.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,31 @@ describe("batches for the user's agent", () => {
await store.close()
})

it("asks about a forum message whose only reply is to its topic's empty first message, not a post's comment", async () => {
const path = join(mkdtempSync(join(tmpdir(), "batches-")), "messages.db")
const store = await openStore({ path })
await store.chats.saveChats(OWNER, [
{ id: "-1", title: "Group", kind: "group", unreadCount: 0, lastMessageAt: null, participantsCount: null },
])
await store.messages.saveMessages(
OWNER,
"-1",
[
{ ...message(1), text: "" },
{ ...message(2, "1"), threadId: "1" },
{ ...message(3, "2"), threadId: "1" },
message(4),
{ ...message(5, "4"), threadId: "4" },
],
{ via: "history" },
)

const batch = await store.conversations.nextBatch(OWNER, "-1", { size: 50 })

expect(ids(batch?.messages ?? [], true)).toEqual([1, 2, 4])
await store.close()
})

it("asks again about a message whose answer went stale, and about nothing once all are answered", async () => {
const { store, path } = await chatOf(3)
await answered(path, 1)
Expand Down
7 changes: 5 additions & 2 deletions src/store/sqlite/batches.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@ export const BATCH_CONTEXT = 50

/**
* Phase 4 plan A2: a live message the agent is asked about — no reply the messenger itself records to a
* message held here, and no current agent answer. A stale answer makes it need the agent again.
* message held here that the build links (`inThread` in conversations/link.ts), and no current agent answer.
* A stale answer makes it need the agent again.
*/
const replied = (m: SQL) => sql`EXISTS (SELECT 1 FROM messages p WHERE p.chat_id = ${m}.chat_id
AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL)`
AND p.external_id = ${m}.reply_to_external_id AND p.deleted_at IS NULL
AND (p.thread_external_id IS ${m}.thread_external_id
OR (p.external_id = ${m}.thread_external_id AND trim(p.text) <> '')))`

const needsAgent = (m: SQL) => sql`(
NOT ${replied(m)}
Expand Down
Loading