fix(knowledge): stabilize skipped connector documents

Author: waleedlatif1

apps/sim/lib/knowledge/connectors/sync-engine.test.ts

@@ -504,14 +504,36 @@ describe('classifyExternalDoc', () => {
     ).toEqual({ type: 'skip', existingId: 'doc-1' })
   })
 
-  it('rehydrates a content-less placeholder even when its listing hash is unchanged', async () => {
+  it('keeps a stable listing-time skipped placeholder unchanged', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(
+      classifyExternalDoc(
+        { ...base, content: '', skippedReason: 'too big' },
+        { id: 'doc-1', contentHash: 'h1', storageKey: null }
+      )
+    ).toEqual({ type: 'unchanged' })
+  })
+
+  it('keeps a stable deferred skipped placeholder unchanged', async () => {
     const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
 
     expect(
       classifyExternalDoc(
         { ...base, content: '', contentDeferred: true },
         { id: 'doc-1', contentHash: 'h1', storageKey: null }
       )
+    ).toEqual({ type: 'unchanged' })
+  })
+
+  it('rehydrates a deferred skipped placeholder carrying an explicit retry hash', async () => {
+    const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine')
+
+    expect(
+      classifyExternalDoc(
+        { ...base, content: '', contentDeferred: true },
+        { id: 'doc-1', contentHash: 'connector:retry:h1', storageKey: null }
+      )
     ).toEqual({ type: 'update', existingId: 'doc-1' })
   })
 

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -454,6 +454,13 @@ type DocClassification =
   | { type: 'unchanged' }
   | { type: 'drop' }
 
+function shouldReplaceWithSkippedState(
+  existing: { storageKey?: string | null },
+  skipped: Pick<ExternalDocument, 'skippedExistingDisposition'>
+): boolean {
+  return existing.storageKey === null || skipped.skippedExistingDisposition === 'replace'
+}
+
 /**
  * Decides what a listed external document becomes during reconciliation.
  *
@@ -462,9 +469,11 @@ type DocClassification =
  *   content stays last-known-good unless the connector marks the skip authoritative.
  * - `drop`: empty, non-deferred content that cannot be indexed.
  * - `add` / `update` / `unchanged`: normal content reconciliation by content hash.
- * - A deferred listing always rehydrates an existing content-less placeholder,
- *   even when its listing hash is unchanged, so a prior hydration-time skip can
- *   recover when the source becomes indexable.
+ * - A deferred listing rehydrates an existing content-less placeholder when its
+ *   listing hash changes. A connector whose skip must be retried independently
+ *   persists `skippedRetryContentHash`, which deliberately differs from the next
+ *   listing hash. Stable permanent skips therefore do not redownload or surface
+ *   as source failures on every sync.
  *
  * `forceRehydrate` (set on a full resync of a `rehydrateOnFullSync` connector) promotes
  * an otherwise-`unchanged` deferred document to `update` so its content is re-fetched —
@@ -487,7 +496,10 @@ export function classifyExternalDoc(
 ): DocClassification {
   if (extDoc.skippedReason) {
     if (!existing) return { type: 'skip' }
-    return existing.storageKey === null || extDoc.skippedExistingDisposition === 'replace'
+    if (existing.storageKey === null && existing.contentHash === extDoc.contentHash) {
+      return { type: 'unchanged' }
+    }
+    return shouldReplaceWithSkippedState(existing, extDoc)
       ? { type: 'skip', existingId: existing.id }
       : { type: 'unchanged' }
   }
@@ -497,9 +509,6 @@ export function classifyExternalDoc(
   if (!existing) {
     return { type: 'add' }
   }
-  if (existing.storageKey === null && extDoc.contentDeferred) {
-    return { type: 'update', existingId: existing.id }
-  }
   if (existing.contentHash !== extDoc.contentHash) {
     return { type: 'update', existingId: existing.id }
   }
@@ -2312,7 +2321,7 @@ export async function executeSync(
           // verified-unchanged match — same as the deferred-hydration
           // equivalent above. A genuine hash match never sets skippedReason,
           // so this only fires for the short-circuited case.
-          if (extDoc.skippedReason && existing) {
+          if (extDoc.skippedReason && existing && existing.storageKey !== null) {
             recordUnverifiedExistingRefresh(result, failedExternalIds, extDoc.externalId)
           } else {
             result.docsUnchanged++
@@ -2377,7 +2386,8 @@ export async function executeSync(
                   extDoc: mergeHydratedSkippedDocument(op.extDoc, fullDoc),
                 })
               } else if (op.type === 'update') {
-                if (fullDoc.skippedExistingDisposition === 'replace') {
+                const existing = priorByExternalId.get(op.extDoc.externalId)
+                if (existing && shouldReplaceWithSkippedState(existing, fullDoc)) {
                   skipOps.push({
                     type: 'skip',
                     existingId: op.existingId,