content-processor.ts

import { Readability } from '@mozilla/readability';
import axios from 'axios';
import { load } from 'cheerio';
import { JSDOM } from 'jsdom';
import puppeteer, { Browser, Page } from 'puppeteer';
import sanitizeHtml from 'sanitize-html';
import TurndownService from 'turndown';
import * as fs from 'fs';
import * as path from 'path';
import { Logger } from './logger';
import { Utils } from './utils';
import {
    SourceConfig,
    WebsiteSourceConfig,
    LocalDirectorySourceConfig,
    CodeSourceConfig,
    DocumentChunk,
    BrokenLink
} from './types';
import type { TokenChunker, Tokenizer } from '@chonkiejs/core';
import { CodeChunker } from './code-chunker';

/**
 * Thrown when the browser cannot be launched at all (even after a retry).
 * Without a browser no page in the crawl can be processed, so this aborts the
 * whole website source — the run is then reported as failed instead of
 * "succeeding" with an error on every URL.
 */
export class BrowserLaunchError extends Error {
    constructor(message: string) {
        super(message);
        this.name = 'BrowserLaunchError';
    }
}

/**
 * Outcome of walking a code source's directory tree.
 *
 * `incomplete` means at least one directory could not be listed, so the set of
 * files the caller saw is only a subset of what is on disk. Callers must not
 * treat the unseen files as deleted: cleanup and last-scanned markers have to
 * be skipped, otherwise a transient I/O error silently purges chunks and the
 * next run never revisits those files.
 */
export interface CodeScanResult {
    processedFiles: number;
    skippedFiles: number;
    maxMtime: number;
    incomplete: boolean;
}

/**
 * Where the browser comes from.
 *
 * Unset (the default): this process launches its own Chromium, which the image
 * has to carry. Set: this process CONNECTS to a browser running elsewhere over
 * the Chrome DevTools Protocol, and the image needs no browser at all. The
 * value is a full CDP WebSocket URL, e.g.
 *   ws://browserless.qdrant.svc.cluster.local:3000?token=<token>
 *
 * A remote browser is the one that keeps the app image small and lets the
 * browser be updated independently of the app (a browser image tracks Chrome
 * stable; a distro chromium is frozen at whatever the distro rebuilt last).
 */
export const BROWSER_WS_ENDPOINT_ENV = 'BROWSER_WS_ENDPOINT';

export function remoteBrowserEndpoint(): string | undefined {
    const raw = process.env[BROWSER_WS_ENDPOINT_ENV]?.trim();
    return raw ? raw : undefined;
}

/**
 * Chromium flags for a browser THIS process launches. They are not sent to a
 * remote browser: launch flags belong to the process that starts the browser.
 */
const LOCAL_LAUNCH_ARGS = [
    '--no-sandbox',
    '--disable-setuid-sandbox',
    '--disable-dev-shm-usage',  // Use /tmp instead of /dev/shm (critical in Docker, default 64MB)
    '--disable-gpu',
    '--disable-extensions',
];

const BROWSER_PROTOCOL_TIMEOUT_MS = 60000;

/**
 * Pages whose uncompressed HTML exceeds this multiple of a source's max_size are
 * skipped before rendering. The max_size check after rendering measures only
 * the main content element, so the whole page is always somewhat larger. The
 * margin keeps pages with a lot of chrome around valid content from being lost.
 * The pages that OOM-killed browserless were 5.6–12.7 MB, against a 1 MB max_size.
 */
export const OVERSIZED_PAGE_FACTOR = 4;

/**
 * Open a browser: connect to the remote one named by BROWSER_WS_ENDPOINT, or
 * launch a local one. Callers close it with browser.close() in both cases —
 * a browserless-style service gives each connection its own browser, so
 * closing ends only this caller's session.
 */
export async function openBrowser(): Promise<Browser> {
    const endpoint = remoteBrowserEndpoint();
    if (endpoint) {
        try {
            return await puppeteer.connect({
                browserWSEndpoint: endpoint,
                protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
            });
        } catch (error) {
            // puppeteer rejects a failed WebSocket handshake with the raw `ws`
            // ErrorEvent (not an Error), which stringifies to "[object Object]".
            // Re-throw a real Error naming the browser host (never the full
            // URL: it carries the token).
            throw new Error(`cannot connect to remote browser at ${describeEndpoint(endpoint)}: ${describeError(error)}`);
        }
    }
    return puppeteer.launch({
        executablePath: await resolveBrowserExecutablePath(),
        args: LOCAL_LAUNCH_ARGS,
        protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
    });
}

/** `ws://host:port` of a CDP endpoint, without path, query (token) or credentials. */
function describeEndpoint(endpoint: string): string {
    try {
        const url = new URL(endpoint);
        return `${url.protocol}//${url.host}`;
    } catch {
        return '<unparseable endpoint>';
    }
}

/**
 * Human-readable message for anything thrown or rejected. Handles Error, the
 * `ws` ErrorEvent shape ({ message, error }) and other plain objects, which
 * String() would render as "[object Object]".
 */
export function describeError(error: unknown): string {
    if (error instanceof Error) return error.message || error.name;
    if (error && typeof error === 'object') {
        const obj = error as { message?: unknown; error?: unknown; type?: unknown };
        if (obj.error instanceof Error && obj.error.message) return obj.error.message;
        if (typeof obj.message === 'string' && obj.message) return obj.message;
        if (typeof obj.type === 'string' && obj.type) return `${obj.type} event`;
        // Deliberately not JSON-serialized: event objects reference the socket,
        // whose URL may carry a token.
        return `non-Error rejection (${error.constructor?.name ?? 'Object'})`;
    }
    return String(error);
}

/**
 * Pauses between browser launch/connect attempts. A remote browser service
 * (browserless) gets a longer window: when its pod is OOMKilled and restarts,
 * it refuses connections for ~30s, and a single 5s retry turned that into
 * several consecutive sources failing outright.
 */
export function browserLaunchRetryDelaysMs(): number[] {
    return remoteBrowserEndpoint() ? [5000, 15000, 30000, 60000] : [5000];
}

/**
 * Resolve which browser binary to launch, in order of preference:
 *  1. PUPPETEER_EXECUTABLE_PATH (explicit override)
 *  2. Puppeteer's own downloaded Chrome for Testing (version-matched, the
 *     reliable choice — Debian's chromium package has shipped builds that
 *     crash at startup in containers, e.g. 150.0.7871 SIGTRAPs immediately)
 *  3. A system chromium, as the fallback (also covers linux/arm64, where
 *     Chrome for Testing is not published)
 *  4. undefined — let puppeteer decide
 */
export async function resolveBrowserExecutablePath(): Promise<string | undefined> {
    const override = process.env.PUPPETEER_EXECUTABLE_PATH;
    if (override) return override;
    try {
        const bundled = await puppeteer.executablePath();
        if (bundled && fs.existsSync(bundled)) return bundled;
    } catch {
        // no browser downloaded for this platform
    }
    if (fs.existsSync('/usr/bin/chromium')) return '/usr/bin/chromium';
    if (fs.existsSync('/usr/bin/chromium-browser')) return '/usr/bin/chromium-browser';
    return undefined;
}

/**
 * Best-effort snapshot of process + container memory, for diagnosing browser
 * launch failures (the usual cause in Kubernetes is the pod's memory cgroup —
 * Chromium forks, then its renderers are immediately OOM-killed).
 */
function memoryDiagnostics(): string {
    const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)}GiB`;
    const parts: string[] = [`node rss ${gib(process.memoryUsage().rss)}`];
    try {
        // cgroup v2, else v1 — absent entirely outside Linux/containers
        let current: string | undefined;
        let max: string | undefined;
        if (fs.existsSync('/sys/fs/cgroup/memory.current')) {
            current = fs.readFileSync('/sys/fs/cgroup/memory.current', 'utf8').trim();
            max = fs.readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim();
        } else if (fs.existsSync('/sys/fs/cgroup/memory/memory.usage_in_bytes')) {
            current = fs.readFileSync('/sys/fs/cgroup/memory/memory.usage_in_bytes', 'utf8').trim();
            max = fs.readFileSync('/sys/fs/cgroup/memory/memory.limit_in_bytes', 'utf8').trim();
        }
        if (current !== undefined && max !== undefined) {
            const unlimited = max === 'max' || Number(max) > 2 ** 60;
            parts.push(`cgroup ${gib(Number(current))} used / ${unlimited ? 'no limit' : gib(Number(max))}`);
        }
    } catch {
        // diagnostics only — never fail because of them
    }
    try {
        // /proc/meminfo reflects the NODE in Kubernetes — catches host-level pressure
        const meminfo = fs.readFileSync('/proc/meminfo', 'utf8');
        const available = meminfo.match(/^MemAvailable:\s+(\d+) kB/m);
        if (available) {
            parts.push(`node available ${gib(Number(available[1]) * 1024)}`);
        }
    } catch {
        // not Linux
    }
    try {
        // Chromium writes its profile and (with --disable-dev-shm-usage) its
        // shared memory under /tmp — a full disk breaks launches
        const stat = fs.statfsSync('/tmp');
        parts.push(`/tmp free ${gib(stat.bavail * stat.bsize)}`);
    } catch {
        // statfs unavailable
    }
    return parts.join(', ');
}

export class ContentProcessor {
    private turndownService: TurndownService;
    private logger: Logger;
    private tokenChunkerCache: Map<string, Promise<TokenChunker>>;
    private codeChunkerCache: Map<string, Promise<CodeChunker>>;
    private tokenizerCache: Promise<Tokenizer> | null;

    constructor(logger: Logger) {
        this.logger = logger;
        this.turndownService = new TurndownService({
            codeBlockStyle: 'fenced',
            headingStyle: 'atx'
        });
        this.tokenChunkerCache = new Map();
        this.codeChunkerCache = new Map();
        this.tokenizerCache = null;
        this.setupTurndownRules();
    }

    private setupTurndownRules() {
        const logger = this.logger.child('markdown');
        logger.debug('Setting up Turndown rules for markdown conversion');
        
        this.turndownService.addRule('codeBlocks', {
            filter: (node: Node): boolean => node.nodeName === 'PRE',
            replacement: (content: string, node: Node): string => {
                const htmlNode = node as HTMLElement;
                const code = htmlNode.querySelector('code');

                let codeContent;
                if (code) {
                    codeContent = code.textContent || '';
                } else {
                    codeContent = htmlNode.textContent || '';
                }

                const lines = codeContent.split('\n');
                let minIndent = Infinity;
                for (const line of lines) {
                    if (line.trim() === '') continue;
                    const leadingWhitespace = line.match(/^\s*/)?.[0] || '';
                    minIndent = Math.min(minIndent, leadingWhitespace.length);
                }

                const cleanedLines = lines.map(line => {
                    return line.substring(minIndent);
                });

                let cleanContent = cleanedLines.join('\n');
                cleanContent = cleanContent.replace(/^\s+|\s+$/g, '');
                cleanContent = cleanContent.replace(/\n{2,}/g, '\n');

                return `\n\`\`\`\n${cleanContent}\n\`\`\`\n`;
            }
        });

        this.turndownService.addRule('tableCell', {
            filter: ['th', 'td'],
            replacement: (content: string, node: Node): string => {
                const htmlNode = node as HTMLElement;

                let cellContent = '';
                if (htmlNode.querySelector('p')) {
                    cellContent = Array.from(htmlNode.querySelectorAll('p'))
                        .map(p => p.textContent || '')
                        .join(' ')
                        .trim();
                } else {
                    cellContent = content.trim();
                }

                return ` ${cellContent.replace(/\|/g, '\\|')} |`;
            }
        });

        this.turndownService.addRule('tableRow', {
            filter: 'tr',
            replacement: (content: string, node: Node): string => {
                const htmlNode = node as HTMLTableRowElement;
                const cells = Array.from(htmlNode.cells);
                const isHeader = htmlNode.parentNode?.nodeName === 'THEAD';

                let output = '|' + content.trimEnd();

                if (isHeader) {
                    const separator = cells.map(() => '---').join(' | ');
                    output += '\n|' + separator + '|';
                }

                if (!isHeader || !htmlNode.nextElementSibling) {
                    output += '\n';
                }

                return output;
            }
        });

        this.turndownService.addRule('table', {
            filter: 'table',
            replacement: (content: string): string => {
                return '\n' + content.replace(/\n+/g, '\n').trim() + '\n';
            }
        });

        this.turndownService.addRule('preserveTableWhitespace', {
            filter: (node: Node): boolean => {
                return (
                    (node.nodeName === 'TD' || node.nodeName === 'TH') &&
                    (node.textContent?.trim().length === 0)
                );
            },
            replacement: (): string => {
                return ' |';
            }
        });
        
        logger.debug('Turndown rules setup complete');
    }

    public convertHtmlToMarkdown(html: string): string {
        if (!html || !html.trim()) {
            return '';
        }

        // Pre-process tabbed content before sanitization strips ARIA attributes
        const dom = new JSDOM(html);
        const tabButtons = dom.window.document.querySelectorAll('[role="tab"]');
        if (tabButtons.length > 0) {
            const logger = this.logger.child('markdown');
            this.preprocessTabs(dom.window.document, logger);
            html = dom.window.document.body.innerHTML;
        }

        // Sanitize the HTML first
        const cleanHtml = sanitizeHtml(html, {
            allowedTags: [
                'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
                'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
                'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td',
                'blockquote', 'br'
            ],
            allowedAttributes: {
                'a': ['href'],
                'pre': ['class', 'data-language'],
                'code': ['class', 'data-language'],
                'div': ['class'],
                'span': ['class']
            }
        });

        // Convert to markdown using TurndownService
        return this.turndownService.turndown(cleanHtml).trim();
    }

    async parseSitemap(sitemapUrl: string, logger: Logger): Promise<Map<string, string | undefined>> {
        logger.info(`Parsing sitemap from ${sitemapUrl}`);
        try {
            const response = await axios.get(sitemapUrl);
            const $ = load(response.data, { xmlMode: true });
            
            const urlMap = new Map<string, string | undefined>();
            
            // Handle standard sitemaps — extract <loc> and optional <lastmod>
            $('url').each((_, element) => {
                const loc = $(element).find('loc').text().trim();
                const lastmod = $(element).find('lastmod').text().trim() || undefined;
                if (loc) {
                    urlMap.set(loc, lastmod);
                }
            });
            
            // Handle sitemap indexes (sitemaps that link to other sitemaps)
            const sitemapLinks: string[] = [];
            $('sitemap > loc').each((_, element) => {
                const nestedSitemapUrl = $(element).text().trim();
                if (nestedSitemapUrl) {
                    sitemapLinks.push(nestedSitemapUrl);
                }
            });
            
            // Recursively process nested sitemaps
            for (const nestedSitemapUrl of sitemapLinks) {
                logger.debug(`Found nested sitemap: ${nestedSitemapUrl}`);
                const nestedMap = await this.parseSitemap(nestedSitemapUrl, logger);
                for (const [url, lastmod] of nestedMap) {
                    urlMap.set(url, lastmod);
                }
            }
            
            const withLastmod = [...urlMap.values()].filter(v => v !== undefined).length;
            logger.info(`Found ${urlMap.size} URLs in sitemap ${sitemapUrl} (${withLastmod} with lastmod)`);
            return urlMap;
        } catch (error) {
            logger.error(`Error parsing sitemap at ${sitemapUrl}:`, error);
            return new Map();
        }
    }

    async crawlWebsite(
        baseUrl: string,
        sourceConfig: WebsiteSourceConfig,
        processPageContent: (url: string, content: string) => Promise<boolean>,
        parentLogger: Logger,
        visitedUrls: Set<string>,
        options?: {
            knownUrls?: Set<string>;
            etagStore?: {
                get: (url: string) => Promise<string | undefined>;
                set: (url: string, etag: string) => Promise<void>;
            };
            lastmodStore?: {
                get: (url: string) => Promise<string | undefined>;
                set: (url: string, lastmod: string) => Promise<void>;
            };
            // When the Postgres markdown store is enabled, this set contains
            // URLs that already have markdown stored.  URLs NOT in this set
            // bypass lastmod/ETag skip logic so that the store is fully
            // populated on the first sync.
            markdownStoreUrls?: Set<string>;
            // When true, bypass all lastmod/ETag skip logic. Used when a
            // previous sync was interrupted or never completed successfully,
            // ensuring every page is processed at least once.
            forceFullSync?: boolean;
        }
    ): Promise<{ hasNetworkErrors: boolean; brokenLinks: BrokenLink[]; notFoundUrls: Set<string> }> {
        const logger = parentLogger.child('crawler');
        const queue: string[] = [baseUrl];
        const brokenLinks: BrokenLink[] = [];
        const brokenLinkKeys: Set<string> = new Set();
        const notFoundUrls: Set<string> = new Set();
        const referrers: Map<string, Set<string>> = new Map();

        const addReferrer = (targetUrl: string, sourceUrl: string) => {
            const existing = referrers.get(targetUrl);
            if (existing) {
                existing.add(sourceUrl);
            } else {
                referrers.set(targetUrl, new Set([sourceUrl]));
            }
        };

        const addBrokenLink = (sourceUrl: string, targetUrl: string) => {
            const key = `${sourceUrl} -> ${targetUrl}`;
            if (brokenLinkKeys.has(key)) return;
            brokenLinkKeys.add(key);
            brokenLinks.push({ source: sourceUrl, target: targetUrl });
        };

        addReferrer(baseUrl, baseUrl);

        const { knownUrls, etagStore, lastmodStore, markdownStoreUrls, forceFullSync } = options ?? {};

        // Pre-seed queue with previously-known URLs so link discovery isn't lost
        // when pages are skipped via ETag/lastmod matching on re-runs
        if (knownUrls && knownUrls.size > 0) {
            for (const url of knownUrls) {
                if (url.startsWith(sourceConfig.url) && !queue.includes(url)) {
                    queue.push(url);
                }
            }
            logger.info(`Pre-seeded ${knownUrls.size} known URLs from database into crawl queue`);
        }

        // Process sitemap if provided — extract URLs and lastmod dates
        // The lastmod map is used later in the crawl loop for change detection
        const sitemapLastmod = new Map<string, string>();
        if (sourceConfig.sitemap_url) {
            logger.section('SITEMAP PROCESSING');
            const sitemapData = await this.parseSitemap(sourceConfig.sitemap_url, logger);
            
            // Collect parent URLs that have lastmod (directories ending with /)
            // so we can inherit lastmod for child URLs that lack their own.
            // Example: /gloo-mesh/2.10.x/ has lastmod but /gloo-mesh/2.10.x/reference/... doesn't.
            const parentLastmods: Array<{ prefix: string; lastmod: string }> = [];
            
            // First pass: add URLs to queue and collect explicit lastmod values
            for (const [url, lastmod] of sitemapData) {
                if (url.startsWith(sourceConfig.url)) {
                    addReferrer(url, sourceConfig.sitemap_url);
                    if (lastmod) {
                        sitemapLastmod.set(url, lastmod);
                        // Track as a potential parent for child URL inheritance
                        if (url.endsWith('/')) {
                            parentLastmods.push({ prefix: url, lastmod });
                        }
                    }
                    if (!queue.includes(url)) {
                        logger.debug(`Adding URL from sitemap to queue: ${url}`);
                        queue.push(url);
                    }
                }
            }
            
            // Second pass: inherit lastmod from the most specific parent for URLs
            // that don't have their own lastmod
            if (parentLastmods.length > 0) {
                // Sort by prefix length descending so longest (most specific) match wins
                parentLastmods.sort((a, b) => b.prefix.length - a.prefix.length);
                let inheritedCount = 0;
                for (const [url] of sitemapData) {
                    if (url.startsWith(sourceConfig.url) && !sitemapLastmod.has(url)) {
                        const parent = parentLastmods.find(p => url.startsWith(p.prefix));
                        if (parent) {
                            sitemapLastmod.set(url, parent.lastmod);
                            inheritedCount++;
                        }
                    }
                }
                if (inheritedCount > 0) {
                    logger.info(`Inherited lastmod from parent URLs for ${inheritedCount} child URLs without their own lastmod`);
                }
            }
            
            logger.info(`Added ${queue.length - 1} URLs from sitemap to the crawl queue (${sitemapLastmod.size} with lastmod)`);
        }

        logger.info(`Starting crawl from ${baseUrl} with ${queue.length} URLs in initial queue`);
        let processedCount = 0;
        let skippedCount = 0;
        let skippedSizeCount = 0;
        let etagSkippedCount = 0;
        let lastmodSkippedCount = 0;
        let pdfProcessedCount = 0;
        let errorCount = 0;
        let hasNetworkErrors = false;
        // Tracks whether the server supports ETags. Starts true and is set to false
        // after a few consecutive successful HEAD responses return no ETag header,
        // avoiding wasted HEAD requests for servers that don't support ETags.
        // HEAD request failures (network errors, timeouts) do NOT count toward
        // this threshold — they're transient issues, not evidence of missing ETag support.
        let serverSupportsEtags = true;
        let consecutiveMissingEtag = 0;
        const MISSING_ETAG_THRESHOLD = 3;

        // Track per-URL retry counts for rate limiting (429) responses
        const retryCounts: Map<string, number> = new Map();
        const MAX_RETRIES_PER_URL = 3;
        const DEFAULT_RETRY_DELAY_MS = 30000; // 30s default when no Retry-After header
        const MAX_RETRY_DELAY_MS = 120000;    // Cap at 2 minutes

        // Adaptive backoff for HEAD requests: starts at 0 (no delay), increases
        // on 429 responses, and decays back toward 0 on successful responses.
        let headBackoffMs = 0;
        const HEAD_BACKOFF_INITIAL_MS = 200;  // First backoff step after a 429
        const HEAD_BACKOFF_MAX_MS = 5000;     // Cap at 5 seconds
        const HEAD_BACKOFF_DECAY_FACTOR = 0.5; // Halve delay on each success
        const HEAD_BACKOFF_FLOOR_MS = 10;     // Below this, snap to 0

        // Launch a single browser instance and reuse one page (tab) for all URLs
        let browser: Browser | null = null;
        let page: Page | null = null;

        const launchBrowser = async (): Promise<{ browser: Browser; page: Page }> => {
            // Retry after pauses — the failure may be transient (memory
            // pressure from a page that just closed, or a remote browser
            // service restarting). If the browser cannot launch at all, no page
            // in this crawl can be processed, so escalate as fatal instead of
            // failing every URL one by one.
            const delays = browserLaunchRetryDelaysMs();
            const attempts = delays.length + 1;
            for (let attempt = 1; ; attempt++) {
                let b: Browser | undefined;
                try {
                    b = await openBrowser();
                    const p = await b.newPage();
                    if (attempt > 1) logger.info(`Browser launched on attempt ${attempt}/${attempts}`);
                    return { browser: b, page: p };
                } catch (error) {
                    // Don't leak a remote session whose first page failed to open
                    if (b) {
                        try { await b.close(); } catch { /* already gone */ }
                    }
                    const reason = describeError(error);
                    if (attempt >= attempts) {
                        throw new BrowserLaunchError(
                            `browser failed to launch after ${attempts} attempts (${memoryDiagnostics()}): ${reason}`
                        );
                    }
                    const delayMs = delays[attempt - 1];
                    logger.warn(`Browser launch failed (attempt ${attempt}/${attempts}, ${memoryDiagnostics()}): ${reason} — retrying in ${Math.round(delayMs / 1000)}s...`);
                    await new Promise(resolve => setTimeout(resolve, delayMs));
                }
            }
        };

        // Track whether the page needs to be recreated (e.g., after a timeout error)
        let pageNeedsRecreation = false;
        // Track consecutive protocol-level errors (indicates browser process is degraded)
        let consecutiveProtocolErrors = 0;
        // Track pages processed since the last browser (re)launch for periodic restarts
        let pagesSinceRestart = 0;
        // Restart the browser every N pages to prevent memory accumulation
        const PAGES_BETWEEN_RESTARTS = 50;

        // Planned restarts (first launch, the every-N-pages recycle) are routine
        // and would otherwise dominate a crawl's warning count; only unplanned
        // ones — a disconnected or degraded browser — deserve a warning.
        const restartBrowser = async (reason: string, planned = false): Promise<void> => {
            if (planned) logger.info(`Restarting browser: ${reason}`);
            else logger.warn(`Restarting browser: ${reason}`);
            if (browser) {
                try { await browser.close(); } catch { /* ignore close errors on a degraded browser */ }
            }
            const launched = await launchBrowser();
            browser = launched.browser;
            page = launched.page;
            pageNeedsRecreation = false;
            consecutiveProtocolErrors = 0;
            pagesSinceRestart = 0;
        };

        const ensureBrowser = async (): Promise<Page> => {
            // If protocol errors are persisting after page recreation, the browser
            // process itself is degraded — a full restart is needed.
            const needsBrowserRestart = pageNeedsRecreation && consecutiveProtocolErrors >= 1;

            if (!browser || !browser.connected || needsBrowserRestart) {
                const reason = needsBrowserRestart
                    ? `protocol errors persisting after page recreation (${consecutiveProtocolErrors} consecutive)`
                    : browser ? 'browser disconnected' : 'initial launch';
                await restartBrowser(reason, !browser && !needsBrowserRestart);
            } else if (pagesSinceRestart >= PAGES_BETWEEN_RESTARTS) {
                // Periodic restart to prevent memory accumulation from long crawls
                await restartBrowser(`periodic restart after ${pagesSinceRestart} pages`, true);
            } else if (pageNeedsRecreation) {
                // The previous page.evaluate or navigation timed out / errored.
                // Close the stale page and create a fresh one to avoid corrupted state.
                logger.info('Recreating page after previous error...');
                try {
                    if (page) await page.close();
                } catch {
                    // Ignore errors closing a stale page
                }
                page = await browser.newPage();
                pageNeedsRecreation = false;
            }
            return page!;
        };

        try {
            while (queue.length > 0) {
                const url = queue.shift();
                if (!url) continue;

                const normalizedUrl = Utils.normalizeUrl(url);
                if (visitedUrls.has(normalizedUrl)) continue;
                visitedUrls.add(normalizedUrl);

                if (!Utils.shouldProcessUrl(url)) {
                    logger.debug(`Skipping URL with unsupported extension: ${url}`);
                    skippedCount++;
                    continue;
                }

                // Sitemap lastmod-based skip: if the sitemap provides a lastmod date for
                // this URL and it matches the stored value, skip entirely. This avoids
                // both HEAD requests and full page loads — the sitemap is fetched once
                // for the entire crawl.
                // Bypass this skip when:
                //   - forceFullSync is true (previous sync never completed)
                //   - markdownStoreUrls is set and the URL is not in it (markdown store gap)
                const urlLastmod = sitemapLastmod.get(url);
                const urlInMarkdownStore = !markdownStoreUrls || markdownStoreUrls.has(url);
                if (urlLastmod && lastmodStore) {
                    const storedLastmod = await lastmodStore.get(url);
                    if (storedLastmod && storedLastmod === urlLastmod) {
                        if (forceFullSync) {
                            logger.info(`Lastmod unchanged but full sync not yet completed, forcing processing: ${url}`);
                        } else if (!urlInMarkdownStore) {
                            logger.info(`Lastmod unchanged but URL not in markdown store, forcing processing: ${url}`);
                        } else {
                            logger.info(`Lastmod unchanged (${urlLastmod}), skipping: ${url}`);
                            visitedUrls.add(url);
                            lastmodSkippedCount++;
                            continue;
                        }
                    } else if (storedLastmod) {
                        logger.debug(`Lastmod changed (stored: ${storedLastmod}, sitemap: ${urlLastmod}), processing: ${url}`);
                    } else {
                        logger.debug(`No stored lastmod for ${url}, processing`);
                    }
                }

                // ETag-based skip: send a lightweight HEAD request and compare the ETag
                // to the stored value. If unchanged, skip the entire page (no Puppeteer,
                // no content extraction, no chunking, no embedding).
                // Skip ETag check if the URL has a lastmod from the sitemap — we trust
                // lastmod as the primary change-detection signal when available.
                if (!urlLastmod && !Utils.isPdfUrl(url) && etagStore && serverSupportsEtags) {
                    // Adaptive backoff: wait before HEAD if we've been rate-limited recently
                    if (headBackoffMs > 0) {
                        await new Promise(resolve => setTimeout(resolve, headBackoffMs));
                    }

                    const headConfig = {
                        timeout: 10000,
                        headers: { 'User-Agent': 'Mozilla/5.0 (compatible; doc2vec crawler)' },
                        maxRedirects: 5,
                    };

                    // Checks an ETag from a HEAD response against the stored value.
                    // Returns 'skip' if unchanged, 'process' if changed/missing, 'no-etag' if header absent.
                    const checkEtag = async (etag: string | undefined): Promise<'skip' | 'process' | 'no-etag'> => {
                        if (!etag) return 'no-etag';
                        consecutiveMissingEtag = 0;
                        const storedEtag = await etagStore.get(url);
                        if (storedEtag && storedEtag === etag) {
                            return 'skip';
                        } else if (storedEtag) {
                            logger.debug(`ETag changed (stored: ${storedEtag}, received: ${etag}), processing: ${url}`);
                        } else {
                            logger.debug(`No stored ETag for ${url}, processing`);
                        }
                        return 'process';
                    };

                    // Decay backoff on successful HEAD (non-429)
                    const decayBackoff = () => {
                        if (headBackoffMs > 0) {
                            headBackoffMs = headBackoffMs * HEAD_BACKOFF_DECAY_FACTOR;
                            if (headBackoffMs < HEAD_BACKOFF_FLOOR_MS) {
                                headBackoffMs = 0;
                                logger.debug('HEAD backoff decayed to 0ms');
                            }
                        }
                    };

                    // Increase backoff on 429
                    const increaseBackoff = () => {
                        const previousMs = headBackoffMs;
                        headBackoffMs = headBackoffMs === 0
                            ? HEAD_BACKOFF_INITIAL_MS
                            : Math.min(headBackoffMs * 2, HEAD_BACKOFF_MAX_MS);
                        logger.debug(`HEAD backoff increased: ${previousMs}ms → ${headBackoffMs}ms`);
                    };

                    try {
                        const headResponse = await axios.head(url, headConfig);
                        decayBackoff();
                        const result = await checkEtag(headResponse.headers['etag']);
                        if (result === 'skip') {
                            // Bypass ETag skip when:
                            //   - forceFullSync is true (previous sync never completed)
                            //   - markdownStoreUrls is set and URL is not in it
                            if (forceFullSync) {
                                logger.info(`ETag unchanged but full sync not yet completed, forcing processing: ${url}`);
                            } else if (!urlInMarkdownStore) {
                                logger.info(`ETag unchanged but URL not in markdown store, forcing processing: ${url}`);
                            } else {
                                logger.info(`ETag unchanged, skipping: ${url}`);
                                visitedUrls.add(url);
                                etagSkippedCount++;
                                continue;
                            }
                        } else if (result === 'no-etag') {
                            consecutiveMissingEtag++;
                            logger.debug(`No ETag in HEAD response for ${url} (${consecutiveMissingEtag}/${MISSING_ETAG_THRESHOLD})`);
                            if (consecutiveMissingEtag >= MISSING_ETAG_THRESHOLD) {
                                serverSupportsEtags = false;
                                logger.warn(`Server does not return ETag headers (${MISSING_ETAG_THRESHOLD} consecutive pages without ETag). Disabling ETag-based skip for this crawl.`);
                            }
                        }
                    } catch (headError: any) {
                        // If the HEAD request was rate-limited (429), increase backoff,
                        // wait, and retry once. The backoff ensures subsequent HEAD
                        // requests are spaced out to avoid further 429s.
                        const headStatus = headError?.response?.status;
                        if (headStatus === 429) {
                            increaseBackoff();
                            const retryAfterMs = this.parseRetryAfter(headError.response?.headers?.['retry-after']);
                            const delayMs = Math.min(retryAfterMs ?? 5000, 30000);
                            logger.debug(`HEAD rate-limited (429) for ${url}, waiting ${Math.round(delayMs / 1000)}s before retry`);
                            await new Promise(resolve => setTimeout(resolve, delayMs));
                            try {
                                const retryResponse = await axios.head(url, headConfig);
                                decayBackoff();
                                const result = await checkEtag(retryResponse.headers['etag']);
                                if (result === 'skip') {
                                    if (forceFullSync) {
                                        logger.info(`ETag unchanged but full sync not yet completed, forcing processing: ${url}`);
                                    } else if (!urlInMarkdownStore) {
                                        logger.info(`ETag unchanged but URL not in markdown store, forcing processing: ${url}`);
                                    } else {
                                        logger.info(`ETag unchanged, skipping: ${url}`);
                                        visitedUrls.add(url);
                                        etagSkippedCount++;
                                        continue;
                                    }
                                }
                                // 'process' or 'no-etag' — fall through to full processing
                            } catch {
                                logger.debug(`HEAD retry also failed for ${url}, falling through to full processing`);
                            }
                        } else if (headStatus && headStatus >= 400 && headStatus < 500 && headStatus !== 405) {
                            // Client error (404, 403, 410, etc.) — the page doesn't
                            // exist or is inaccessible. Skip instead of wasting a
                            // full Puppeteer load on a non-existent page.
                            // 405 (Method Not Allowed) is excluded because the server
                            // may not support HEAD but the page may still exist via GET.
                            logger.debug(`HEAD returned ${headStatus} for ${url}, skipping`);
                            if (headStatus === 404) {
                                notFoundUrls.add(url);
                                // The URL was optimistically added to visitedUrls when
                                // dequeued (before we knew it was dead). Remove it so the
                                // obsolete-chunk cleanup treats it as gone and purges its
                                // chunks — otherwise deleted pages linger forever.
                                visitedUrls.delete(normalizedUrl);
                            }
                            skippedCount++;
                            continue;
                        } else {
                            // Non-HTTP failure (network error, timeout), 5xx server
                            // error, or 405 — fall through to full processing.
                            // Do NOT count toward the missing-ETag threshold.
                            logger.debug(`HEAD request failed for ${url}: ${headError?.message || headError}. Falling through to full processing.`);
                        }
                    }
                }

                try {
                    logger.info(`Crawling: ${url}`);
                    const sources = referrers.get(url) ?? new Set([baseUrl]);

                    // A page far larger than max_size would be discarded after
                    // rendering anyway, and rendering multi-MB pages is what
                    // OOM-kills the browser. Skip it before Chromium sees it, but
                    // still harvest its links from the raw HTML so the crawl
                    // discovers the same pages as before.
                    const oversized = Utils.isPdfUrl(url)
                        ? null
                        : await this.probeOversizedPage(url, sourceConfig.max_size, logger);

                    // For HTML pages, ensure the browser is running and pass the shared page
                    // For PDFs, processPage handles them without Puppeteer
                    let currentPage: Page | undefined;
                    let result: { content: string | null, links: string[], finalUrl: string, etag?: string };
                    if (oversized) {
                        logger.warn(`Page is ${oversized.bytes} bytes, over ${OVERSIZED_PAGE_FACTOR}x max size (${sourceConfig.max_size}). Skipping without rendering: ${url}`);
                        const raw = await this.extractLinksWithoutBrowser(oversized.finalUrl, logger);
                        result = { content: null, links: raw.links, finalUrl: raw.finalUrl };
                    } else {
                        currentPage = Utils.isPdfUrl(url) ? undefined : await ensureBrowser();
                        result = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
                            if (status === 404) {
                                for (const source of sources) {
                                    addBrokenLink(source, reportedUrl);
                                }
                            }
                        }, currentPage);
                    }

                    let contentProcessedSuccessfully = false;
                    if (result.content !== null) {
                        contentProcessedSuccessfully = await processPageContent(url, result.content);
                        if (Utils.isPdfUrl(url)) {
                            pdfProcessedCount++;
                        } else {
                            processedCount++;
                        }
                    } else {
                        skippedSizeCount++;
                    }

                    // Use links extracted from the full rendered DOM by processPage
                    // (no separate axios request needed)
                    if (result.links.length > 0) {
                        const pageUrlForLinks = result.finalUrl || url;
                        logger.debug(`Finding links on page ${url}`);
                        let newLinksFound = 0;

                        for (const href of result.links) {
                            const fullUrl = Utils.buildUrl(href, pageUrlForLinks, logger);
                            if (fullUrl.startsWith(sourceConfig.url)) {
                                addReferrer(fullUrl, pageUrlForLinks);
                                if (!visitedUrls.has(Utils.normalizeUrl(fullUrl))) {
                                    if (!queue.includes(fullUrl)) {
                                        queue.push(fullUrl);
                                        newLinksFound++;
                                    }
                                }
                            }
                        }

                        logger.debug(`Found ${newLinksFound} new links on ${url}`);
                    }

                    // Only store ETag/lastmod when content was processed successfully.
                    // If chunking or embedding failed, we want the next run to retry
                    // this page rather than skipping it based on stale metadata.
                    if (contentProcessedSuccessfully) {
                        // Store the ETag for this URL so subsequent runs can skip it
                        if (result.etag && etagStore) {
                            try {
                                await etagStore.set(url, result.etag);
                            } catch {
                                // ETag storage failure is non-critical
                            }
                        }

                        // Store the sitemap lastmod for this URL so subsequent runs can
                        // skip it without any HTTP requests when the lastmod hasn't changed
                        if (urlLastmod && lastmodStore) {
                            try {
                                await lastmodStore.set(url, urlLastmod);
                            } catch {
                                // Lastmod storage failure is non-critical
                            }
                        }
                    }

                    // Page processed successfully — reset protocol error counter and
                    // increment the periodic restart counter
                    consecutiveProtocolErrors = 0;
                    if (currentPage) {
                        pagesSinceRestart++;
                    }

                    // Navigate to about:blank to clear any lingering JS, timers, or
                    // event listeners from the processed page before moving to the next URL.
                    if (currentPage) {
                        try {
                            await currentPage.goto('about:blank', { waitUntil: 'load', timeout: 5000 });
                        } catch {
                            // If even about:blank fails, the page is stuck — mark for recreation
                            pageNeedsRecreation = true;
                        }
                    }
                } catch (error: any) {
                    // A browser that cannot launch is fatal for the whole crawl —
                    // rethrow so the source (and the run) is reported as failed
                    // rather than logging an error per URL and finishing "green".
                    // Same for a permanent embedding failure (bad API key):
                    // every remaining page would fail identically.
                    if (error instanceof BrowserLaunchError || error?.name === 'FatalEmbeddingError') {
                        logger.error(`Aborting crawl of ${baseUrl}: ${error.message}`);
                        throw error;
                    }

                    const status = this.getHttpStatus(error);

                    // ── Handle 429 (Too Many Requests) with retry ──
                    if (status === 429) {
                        const retries = retryCounts.get(url) ?? 0;
                        if (retries < MAX_RETRIES_PER_URL) {
                            const retryAfterMs = error.retryAfterMs as number | undefined;
                            const delayMs = Math.min(
                                retryAfterMs ?? DEFAULT_RETRY_DELAY_MS,
                                MAX_RETRY_DELAY_MS
                            );
                            retryCounts.set(url, retries + 1);

                            logger.warn(
                                `Rate limited (429) on ${url} (attempt ${retries + 1}/${MAX_RETRIES_PER_URL}). ` +
                                `Waiting ${Math.round(delayMs / 1000)}s before retry...`
                            );

                            await new Promise(resolve => setTimeout(resolve, delayMs));

                            // Re-add to the front of the queue so it's retried next.
                            // Remove from visitedUrls so it's not skipped.
                            visitedUrls.delete(normalizedUrl);
                            queue.unshift(url);

                            // Do NOT set pageNeedsRecreation — the page/browser is healthy,
                            // the server is just rate-limiting us.
                            // Do NOT increment errorCount — this is a temporary condition.
                            continue;
                        }

                        // Max retries exhausted — fall through to normal error handling
                        logger.error(`Rate limit retries exhausted for ${url} after ${MAX_RETRIES_PER_URL} attempts`);
                    }

                    // ── General error handling ──
                    logger.error(`Failed during processing or link discovery for ${url}:`, error);
                    errorCount++;

                    // Mark the page for recreation so the next URL gets a clean tab.
                    // This prevents cascading failures from a stuck/corrupted page state.
                    pageNeedsRecreation = true;

                    // Track protocol-level errors separately — these indicate the browser
                    // process itself is degraded, not just the page. If they persist after
                    // page recreation, ensureBrowser() will escalate to a full browser restart.
                    if (this.isProtocolError(error)) {
                        consecutiveProtocolErrors++;
                        logger.warn(`Protocol error detected (${consecutiveProtocolErrors} consecutive), browser may need restart`);
                    } else {
                        consecutiveProtocolErrors = 0;
                    }

                    if (status === 404) {
                        notFoundUrls.add(url);
                        // The URL was optimistically added to visitedUrls when dequeued
                        // (before this GET revealed it's a 404). Remove it so the
                        // obsolete-chunk cleanup treats it as gone and purges its chunks.
                        visitedUrls.delete(normalizedUrl);
                        const sources = referrers.get(url) ?? new Set([baseUrl]);
                        for (const source of sources) {
                            addBrokenLink(source, url);
                        }
                    }
                    
                    // Check if this is a network error (DNS resolution, connection issues, etc.)
                    if (this.isNetworkError(error)) {
                        hasNetworkErrors = true;
                        logger.warn(`Network error detected for ${url}, this may affect cleanup decisions`);
                    }
                }
            }
        } finally {
            // Close the shared browser instance when the crawl is done
            const browserToClose = browser as Browser | null;
            if (browserToClose && browserToClose.connected) {
                await browserToClose.close();
                logger.debug('Shared browser closed after crawl completed');
            }
        }

        logger.info(`Crawl completed. HTML Pages: ${processedCount}, PDFs: ${pdfProcessedCount}, Skipped (Lastmod): ${lastmodSkippedCount}, Skipped (ETag): ${etagSkippedCount}, Skipped (Extension): ${skippedCount}, Skipped (Size): ${skippedSizeCount}, Errors: ${errorCount}`);
        
        if (hasNetworkErrors) {
            logger.warn('Network errors were encountered during crawling. Cleanup may be skipped to avoid removing valid chunks.');
        }
        
        return { hasNetworkErrors, brokenLinks, notFoundUrls };
    }

    /**
     * Detects Chrome DevTools Protocol-level errors that indicate the browser
     * process is degraded/stuck (not just a single page issue).
     * These errors cannot be fixed by page recreation alone — the browser needs a full restart.
     */
    private isProtocolError(error: any): boolean {
        const msg = error?.message || '';
        return msg.includes('ProtocolError') ||
            msg.includes('Protocol error') ||
            (msg.includes('timed out') && msg.includes('protocolTimeout')) ||
            msg.includes('Target closed') ||
            msg.includes('Session closed') ||
            msg.includes('Network.enable timed out') ||
            msg.includes('Connection closed');
    }

    private isNetworkError(error: any): boolean {
        // Check for common network error patterns
        if (error?.code) {
            // DNS resolution errors
            if (error.code === 'ENOTFOUND') return true;
            // Connection refused
            if (error.code === 'ECONNREFUSED') return true;
            // Connection timeout
            if (error.code === 'ETIMEDOUT') return true;
            // Connection reset
            if (error.code === 'ECONNRESET') return true;
            // Host unreachable
            if (error.code === 'EHOSTUNREACH') return true;
            // Network unreachable
            if (error.code === 'ENETUNREACH') return true;
        }
        
        // Check for axios-specific network errors
        if (error?.isAxiosError) {
            // If there's no response, it's likely a network error
            if (!error.response) return true;
        }
        
        // Check error message for network-related terms
        const errorMessage = error?.message?.toLowerCase() || '';
        if (errorMessage.includes('getaddrinfo') || 
            errorMessage.includes('network') || 
            errorMessage.includes('timeout') ||
            errorMessage.includes('connection') ||
            errorMessage.includes('dns')) {
            return true;
        }
        
        return false;
    }

    /**
     * HEAD the page asking for an UNCOMPRESSED body and report its size when it
     * is more than OVERSIZED_PAGE_FACTOR x maxSize. Compressed lengths are
     * useless here (a 7 MB docs page is 27 KB of brotli), so a response that is
     * still encoded, has no Content-Length, fails, or isn't 2xx returns null and
     * the page is rendered as usual.
     */
    protected async probeOversizedPage(
        url: string,
        maxSize: number,
        logger: Logger
    ): Promise<{ bytes: number; finalUrl: string } | null> {
        try {
            const response = await axios.request({
                method: 'HEAD',
                url,
                timeout: 10000,
                maxRedirects: 5,
                headers: {
                    'User-Agent': 'Mozilla/5.0 (compatible; doc2vec crawler)',
                    'Accept-Encoding': 'identity',
                },
                validateStatus: () => true,
                // Otherwise axios deletes Content-Encoding from HEAD responses,
                // hiding that a server ignored `identity` and sent a compressed length
                decompress: false,
            });
            if (response.status < 200 || response.status >= 300) return null;
            const encoding = String(response.headers['content-encoding'] ?? 'identity').toLowerCase();
            if (encoding !== 'identity') return null;
            const bytes = Number(response.headers['content-length']);
            if (!Number.isFinite(bytes) || bytes <= maxSize * OVERSIZED_PAGE_FACTOR) return null;
            const finalUrl: string = (response.request as any)?.res?.responseUrl || url;
            return { bytes, finalUrl };
        } catch (error) {
            logger.debug(`Size probe failed for ${url}: ${describeError(error)}. Rendering normally.`);
            return null;
        }
    }

    /**
     * Collect a page's links from its raw HTML, without a browser. Used for
     * pages too large to render, so skipping them doesn't cut off the pages
     * only they link to. JS-inserted links are missed — acceptable for the
     * static documentation pages that reach this size.
     */
    protected async extractLinksWithoutBrowser(
        url: string,
        logger: Logger
    ): Promise<{ links: string[]; finalUrl: string }> {
        try {
            const response = await axios.get<string>(url, {
                timeout: 60000,
                maxRedirects: 5,
                responseType: 'text',
                maxContentLength: 64 * 1024 * 1024,
                headers: { 'User-Agent': 'Mozilla/5.0 (compatible; doc2vec crawler)' },
            });
            const $ = load(response.data);
            const links: string[] = [];
            $('a[href]').each((_, el) => {
                const href = $(el).attr('href');
                if (href && !href.startsWith('#') && !href.startsWith('mailto:')) {
                    links.push(href);
                }
            });
            logger.debug(`Extracted ${links.length} links from raw HTML of ${url}`);
            const finalUrl: string = (response.request as any)?.res?.responseUrl || url;
            return { links, finalUrl };
        } catch (error) {
            logger.warn(`Could not fetch ${url} to extract its links: ${describeError(error)}`);
            return { links: [], finalUrl: url };
        }
    }

    async processPage(
        url: string,
        sourceConfig: SourceConfig,
        onHttpStatus?: (url: string, status: number) => void,
        existingPage?: Page
    ): Promise<{ content: string | null, links: string[], finalUrl: string, etag?: string }> {
        const logger = this.logger.child('page-processor');
        logger.debug(`Processing content from ${url}`);

        // Check if this is a PDF URL
        if (Utils.isPdfUrl(url)) {
            logger.info(`Processing PDF: ${url}`);
            try {
                const markdown = await this.downloadAndConvertPdfFromUrl(url, logger);
                
                // Check size limit for PDF content
                if (markdown.length > sourceConfig.max_size) {
                    logger.warn(`PDF content (${markdown.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping ${url}.`);
                    return { content: null, links: [], finalUrl: url };
                }
                
                return { content: markdown, links: [], finalUrl: url };
            } catch (error) {
                const status = this.getHttpStatus(error);
                if (status !== undefined && status >= 400) {
                    if (onHttpStatus) {
                        onHttpStatus(url, status);
                    }
                    throw error;
                }
                logger.error(`Failed to process PDF ${url}:`, error);
                return { content: null, links: [], finalUrl: url };
            }
        }

        // HTML page processing logic
        // If an existing page (tab) is provided, reuse it; otherwise launch a standalone browser
        let browser: Browser | null = null;
        let page: Page;
        const ownsTheBrowser = !existingPage;
        try {
            if (existingPage) {
                page = existingPage;
            } else {
                // Standalone mode: open a browser for this single page
                browser = await openBrowser();
                page = await browser.newPage();
            }

            logger.debug(`Navigating to ${url}`);
            const response = await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
            const status = response?.status();
            if (status !== undefined && status >= 400) {
                const error = new Error(`Failed to load page: HTTP ${status}`);
                (error as any).status = status;

                // For 429 (Too Many Requests), extract the Retry-After header so the
                // caller can wait the right amount of time before retrying.
                if (status === 429 && response) {
                    const retryAfterMs = this.parseRetryAfter(response.headers()['retry-after']);
                    if (retryAfterMs !== undefined) {
                        (error as any).retryAfterMs = retryAfterMs;
                        logger.info(`Rate limited (429). Retry-After: ${retryAfterMs}ms`);
                    } else {
                        logger.info(`Rate limited (429). No Retry-After header present.`);
                    }
                }

                throw error;
            }

            // Get the final URL after any redirects
            const finalUrl = page.url();

            // Extract ETag header for change detection on subsequent runs
            const etag = response?.headers()['etag'] || undefined;

            // Extract ALL links from the full rendered DOM before any content filtering
            // This searches the entire document, not just the main content area
            const links: string[] = await this.evaluateWithTimeout(page, () => {
                const anchors = document.querySelectorAll('a[href]');
                const hrefs: string[] = [];
                anchors.forEach(a => {
                    const href = a.getAttribute('href');
                    if (href && !href.startsWith('#') && !href.startsWith('mailto:')) {
                        hrefs.push(href);
                    }
                });
                return hrefs;
            });
            logger.debug(`Extracted ${links.length} links from full DOM of ${url}`);

            const htmlContent: string = await this.evaluateWithTimeout(page, () => {
                // Try specific content selectors first, then fall back to broader ones
                const mainContentElement = 
                    document.querySelector('.docs-content') ||        // Common docs pattern
                    document.querySelector('.doc-content') ||         // Alternative docs pattern
                    document.querySelector('.markdown-body') ||       // GitHub-style
                    document.querySelector('article') ||              // Semantic article
                    document.querySelector('div[role="main"].document') || 
                    document.querySelector('main') || 
                    document.body;
                return mainContentElement.innerHTML;
            });

            if (htmlContent.length > sourceConfig.max_size) {
                logger.warn(`Raw HTML content (${htmlContent.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping detailed processing for ${url}.`);
                return { content: null, links, finalUrl, etag };
            }

            logger.debug(`Got HTML content (${htmlContent.length} chars), creating DOM`);
            const dom = new JSDOM(htmlContent);
            const document = dom.window.document;

            document.querySelectorAll('pre').forEach((pre: HTMLElement) => {
                pre.classList.add('article-content');
                pre.setAttribute('data-readable-content-score', '100');
                this.markCodeParents(pre.parentElement);
            });

            // Pre-process tabbed content: inject tab labels into panels
            // and make hidden panels visible so Readability doesn't discard them.
            // Done in JSDOM (not in Puppeteer's page.evaluate) to avoid triggering
            // reactive framework re-renders when modifying data-state attributes.
            this.preprocessTabs(document, logger);

            // Extract H1s BEFORE Readability - it often strips them as "chrome"
            // We'll inject them back after Readability processing
            const h1Elements = document.querySelectorAll('h1');
            const extractedH1s: string[] = [];
            logger.debug(`[Readability Debug] Found ${h1Elements.length} H1 elements before Readability`);
            h1Elements.forEach((h1: Element, index: number) => {
                const h1Text = h1.textContent?.trim() || '';
                // Skip empty H1s or icon-only H1s (like "link" anchors)
                if (h1Text && h1Text.length > 3 && !h1Text.match(/^(link|#|menu|close)$/i)) {
                    extractedH1s.push(h1Text);
                    logger.debug(`[Readability Debug] Extracted H1[${index}]: "${h1Text.substring(0, 50)}..."`);
                }
                h1.classList.add('original-h1');
            });

            logger.debug(`Applying Readability to extract main content`);
            const reader = new Readability(document, {
                charThreshold: 20,
                classesToPreserve: ['article-content', 'original-h1'],
            });
            const article = reader.parse();

            if (!article || article.content == null) {
                logger.warn(`Failed to parse article content with Readability for ${url}`);
                return { content: null, links, finalUrl, etag };
            }
            
            // Debug: Log what Readability extracted
            logger.debug(`[Readability Debug] article.title: "${article.title}"`);
            logger.debug(`[Readability Debug] article.content length: ${article.content?.length}`);
            logger.debug(`[Readability Debug] article.content starts with: "${article.content?.substring(0, 200)}..."`);
            logger.debug(`[Readability Debug] Contains H1 tag: ${article.content?.includes('<h1')}`);
            logger.debug(`[Readability Debug] Contains H2 tag: ${article.content?.includes('<h2')}`);
            logger.debug(`[Readability Debug] Contains original-h1 class: ${article.content?.includes('original-h1')}`);

            // Restore H1s: find elements with our marker class and convert back from H2
            const articleDom = new JSDOM(article.content);
            const articleDoc = articleDom.window.document;
            const originalH1Elements = articleDoc.querySelectorAll('.original-h1');
            logger.debug(`[Readability Debug] Found ${originalH1Elements.length} elements with .original-h1 class to restore`);
            originalH1Elements.forEach((heading: Element, index: number) => {
                logger.debug(`[Readability Debug] Restoring[${index}]: tagName=${heading.tagName}, text="${heading.textContent?.trim().substring(0, 50)}..."`);
                // Create a new H1 element with the same content
                const h1 = articleDoc.createElement('h1');
                h1.innerHTML = heading.innerHTML;
                // Copy other attributes except class
                Array.from(heading.attributes).forEach(attr => {
                    if (attr.name !== 'class') {
                        h1.setAttribute(attr.name, attr.value);
                    }
                });
                heading.replaceWith(h1);
            });
            const restoredContent = articleDoc.body.innerHTML;
            logger.debug(`[Readability Debug] Restored content contains H1: ${restoredContent.includes('<h1')}`);

            logger.debug(`Sanitizing HTML (${restoredContent.length} chars)`);
            const cleanHtml = sanitizeHtml(restoredContent, {
                 allowedTags: [
                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
                ],
                allowedAttributes: {
                    'a': ['href'],
                    'pre': ['class', 'data-language'],
                    'code': ['class', 'data-language'],
                    'div': ['class'],
                    'span': ['class']
                }
            });

            logger.debug(`Converting HTML to Markdown`);
            let markdown = this.turndownService.turndown(cleanHtml);
            
            // Inject extracted H1s back if they're not in the markdown
            // Readability often strips them as "page chrome"
            // Use article.title as fallback if no H1 was extracted
            const pageTitle = extractedH1s.length > 0 ? extractedH1s[0] : (article.title?.trim() || '');
            if (pageTitle) {
                // Check if markdown already starts with this exact H1 (allowing for leading whitespace)
                const normalizedTitle = pageTitle.replace(/\s+/g, ' ');
                const markdownFirstLine = markdown.trimStart().split('\n')[0] || '';
                const existingH1Match = markdownFirstLine.match(/^#\s+(.+)$/);
                const existingH1Text = existingH1Match ? existingH1Match[1].replace(/\s+/g, ' ').trim() : '';
                
                // Only inject if markdown doesn't already start with this H1
                if (!existingH1Match || existingH1Text !== normalizedTitle) {
                    markdown = `# ${pageTitle}\n\n${markdown}`;
                    logger.debug(`[Readability Debug] Injected page title as H1: "${pageTitle}"`);
                } else {
                    logger.debug(`[Readability Debug] H1 "${pageTitle}" already present in markdown`);
                }
            }
            
            logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
            return { content: markdown, links, finalUrl, etag };
        } catch (error) {
            const status = this.getHttpStatus(error);
            if (status !== undefined && status >= 400) {
                if (onHttpStatus) {
                    onHttpStatus(url, status);
                }
                throw error;
            }
            logger.error(`Error processing page ${url}:`, error);
            return { content: null, links: [], finalUrl: url };
        } finally {
            // Only close the browser if we launched it ourselves (standalone mode)
            if (ownsTheBrowser && browser && browser.connected) {
                 await browser.close();
                 logger.debug(`Browser closed for ${url}`);
            }
        }
    }

    /**
     * Wraps a page.evaluate call with a timeout to prevent indefinite hangs.
     * Some pages have heavy/infinite JavaScript that blocks evaluate calls.
     */
    private evaluateWithTimeout<T>(page: Page, fn: () => T, timeoutMs: number = 30000): Promise<T> {
        return Promise.race([
            page.evaluate(fn),
            new Promise<never>((_, reject) =>
                setTimeout(() => reject(new Error(`page.evaluate timed out after ${timeoutMs}ms`)), timeoutMs)
            )
        ]);
    }

    private getHttpStatus(error: any): number | undefined {
        if (typeof error?.status === 'number') {
            return error.status;
        }

        if (typeof error?.response?.status === 'number') {
            return error.response.status;
        }

        return undefined;
    }

    /**
     * Parses a Retry-After header value into milliseconds.
     * 
     * The header can be either:
     * - A number of seconds (e.g., "60")
     * - An HTTP-date (e.g., "Thu, 15 Feb 2026 14:21:00 GMT")
     * 
     * Returns undefined if the header is missing, empty, or unparseable.
     * Returns a minimum of 1000ms even if the header indicates 0 or a past date.
     */
    parseRetryAfter(headerValue: string | undefined): number | undefined {
        if (!headerValue) return undefined;

        // Try parsing as a number of seconds first
        const seconds = Number(headerValue);
        if (!isNaN(seconds) && isFinite(seconds)) {
            return Math.max(1000, Math.round(seconds * 1000));
        }

        // Try parsing as an HTTP-date
        const date = new Date(headerValue);
        if (!isNaN(date.getTime())) {
            const delayMs = date.getTime() - Date.now();
            return Math.max(1000, delayMs);
        }

        return undefined;
    }

    /**
     * Pre-processes tabbed content in the DOM before Readability runs.
     * 
     * Detects tabs using the standard WAI-ARIA tabs pattern:
     * - Tab buttons: elements with role="tab" and aria-controls pointing to a panel
     * - Tab panels: elements with role="tabpanel" matched by id
     * 
     * For each tab/panel pair, injects the tab label as a bold heading into
     * the panel content, so the context is preserved after conversion to markdown.
     * Also ensures hidden panels are visible so Readability doesn't discard them.
     * 
     * Falls back to positional matching (nth tab -> nth panel) when aria-controls
     * is missing.
     */
    private preprocessTabs(document: Document, logger: Logger) {
        // Find all tab buttons using the WAI-ARIA role="tab" attribute
        const tabButtons = document.querySelectorAll('[role="tab"]');
        if (tabButtons.length === 0) return;

        logger.debug(`[Tab Preprocessing] Found ${tabButtons.length} tab buttons`);

        // Collect all tabpanels for fallback positional matching
        const allPanels = document.querySelectorAll('[role="tabpanel"]');

        // Track panels that have already been labeled to avoid duplicates.
        // Pages can have multiple tab groups that reuse the same panel IDs
        // (e.g., two separate tab sets both using tabs-panel-0, tabs-panel-1, etc.),
        // causing getElementById to return the same panel for different tab buttons.
        const labeledPanels = new Set<Element>();

        tabButtons.forEach((tab: Element, index: number) => {
            const label = tab.textContent?.trim();
            if (!label) {
                logger.debug(`[Tab Preprocessing] Skipping tab[${index}] with empty label`);
                return;
            }

            // Try to find the linked panel via aria-controls -> id
            let panel: Element | null = null;
            const controlsId = tab.getAttribute('aria-controls');
            if (controlsId) {
                panel = document.getElementById(controlsId);
            }

            // Fallback: positional matching (nth tab -> nth panel)
            if (!panel && index < allPanels.length) {
                panel = allPanels[index];
                logger.debug(`[Tab Preprocessing] Using positional fallback for tab[${index}]: "${label}"`);
            }

            if (!panel) {
                logger.debug(`[Tab Preprocessing] No panel found for tab[${index}]: "${label}"`);
                return;
            }

            // Skip if this panel has already been labeled (duplicate panel ID across tab groups)
            if (labeledPanels.has(panel)) {
                logger.debug(`[Tab Preprocessing] Skipping tab[${index}]: "${label}" — panel already labeled`);
                return;
            }
            labeledPanels.add(panel);

            logger.debug(`[Tab Preprocessing] Injecting label for tab[${index}]: "${label}"`);

            // Inject the tab label as bold text at the top of the panel
            const labelElement = document.createElement('p');
            const strong = document.createElement('strong');
            strong.textContent = label + ':';
            labelElement.appendChild(strong);
            panel.insertBefore(labelElement, panel.firstChild);

            // Ensure the panel is visible so Readability doesn't skip it.
            // Remove common hiding patterns:
            // 1. data-state attribute (used by Hextra, Radix, etc.)
            if (panel.getAttribute('data-state') !== 'selected') {
                panel.setAttribute('data-state', 'selected');
            }
            // 2. CSS classes that hide content
            const classList = panel.classList;
            // Common hiding classes across frameworks
            const hidingClasses = ['hidden', 'hx-hidden', 'is-hidden', 'display-none', 'd-none', 'invisible'];
            hidingClasses.forEach(cls => classList.remove(cls));
            // Also remove any class containing 'hidden' as a segment (e.g., 'hx-hidden')
            Array.from(classList).forEach(cls => {
                if (/\bhidden\b/i.test(cls)) {
                    classList.remove(cls);
                }
            });
            // 3. Inline style hiding
            const style = panel.getAttribute('style') || '';
            if (style.includes('display') && style.includes('none')) {
                panel.setAttribute('style', style.replace(/display\s*:\s*none\s*;?/gi, ''));
            }

            // Mark the panel as content so Readability preserves it
            panel.classList.add('article-content');
            panel.setAttribute('data-readable-content-score', '100');
        });

        // Remove the tab buttons since we've injected the labels into panels
        // This prevents the tab labels from appearing twice in the output
        tabButtons.forEach((tab: Element) => {
            tab.remove();
        });

        logger.debug(`[Tab Preprocessing] Completed tab preprocessing`);
    }

    private markCodeParents(node: Element | null) {
        if (!node) return;

        if (node.querySelector('pre, code')) {
            node.classList.add('article-content');
            node.setAttribute('data-readable-content-score', '100');
        }
        this.markCodeParents(node.parentElement);
    }

    private async convertDocToMarkdown(filePath: string, logger: Logger): Promise<string> {
        logger.debug(`Converting DOC to markdown: ${filePath}`);
        
        try {
            // Dynamic import for word-extractor
            const WordExtractor = (await import('word-extractor')).default;
            const extractor = new WordExtractor();
            
            const extracted = await extractor.extract(filePath);
            const text = extracted.getBody();
            
            // Create markdown with filename as title
            let markdown = `# ${path.basename(filePath, '.doc')}\n\n`;
            
            // Clean up the text and add to markdown
            const cleanedText = text
                .replace(/\r\n/g, '\n')  // Normalize line endings
                .replace(/\n{3,}/g, '\n\n')  // Remove excessive line breaks
                .trim();
            
            markdown += cleanedText;
            
            logger.debug(`Converted DOC to ${markdown.length} characters of markdown`);
            return markdown;
            
        } catch (error) {
            logger.error(`Failed to convert DOC ${filePath}:`, error);
            throw error;
        }
    }

    private async convertDocxToMarkdown(filePath: string, logger: Logger): Promise<string> {
        logger.debug(`Converting DOCX to markdown: ${filePath}`);
        
        try {
            // Dynamic import for mammoth
            const mammoth = await import('mammoth');
            
            const result = await mammoth.convertToHtml({ path: filePath });
            const html = result.value;
            
            // Log any warnings from mammoth
            if (result.messages.length > 0) {
                logger.debug(`Mammoth warnings: ${result.messages.map(m => m.message).join(', ')}`);
            }
            
            // Create markdown with filename as title
            let markdown = `# ${path.basename(filePath, '.docx')}\n\n`;
            
            // Convert HTML to Markdown using turndown
            const cleanHtml = sanitizeHtml(html, {
                allowedTags: [
                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'br'
                ],
                allowedAttributes: {
                    'a': ['href'],
                    'pre': ['class'],
                    'code': ['class']
                }
            });
            
            const convertedContent = this.turndownService.turndown(cleanHtml);
            markdown += convertedContent;
            
            // Clean up excessive line breaks
            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
            
            logger.debug(`Converted DOCX to ${markdown.length} characters of markdown`);
            return markdown;
            
        } catch (error) {
            logger.error(`Failed to convert DOCX ${filePath}:`, error);
            throw error;
        }
    }

    private async convertPdfToMarkdown(filePath: string, logger: Logger): Promise<string> {
        logger.debug(`Converting PDF to markdown: ${filePath}`);
        
        try {
            // Dynamic import for PDF.js to handle ES module compatibility
            const pdfjsLib = await import('pdfjs-dist/legacy/build/pdf.mjs');
            
            // Read the PDF file as a buffer and convert to Uint8Array
            const pdfBuffer = fs.readFileSync(filePath);
            const pdfData = new Uint8Array(pdfBuffer);
            
            // Load the PDF document
            const loadingTask = pdfjsLib.getDocument({
                data: pdfData,
                // Disable worker to avoid issues in Node.js environment
                useWorkerFetch: false,
                isEvalSupported: false,
                useSystemFonts: true
            });
            
            const pdfDocument = await loadingTask.promise;
            const numPages = pdfDocument.numPages;
            
            logger.debug(`PDF has ${numPages} pages`);
            
            let markdown = `# ${path.basename(filePath, '.pdf')}\n\n`;
            
            // Extract text from each page
            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
                const page = await pdfDocument.getPage(pageNum);
                const textContent = await page.getTextContent();
                
                // Combine text items into a readable format
                let pageText = '';
                let currentY = -1;
                
                for (const item of textContent.items) {
                    if ('str' in item) {
                        // If this is a new line (different Y position), add a line break
                        if (currentY !== -1 && Math.abs(item.transform[5] - currentY) > 5) {
                            pageText += '\n';
                        }
                        
                        pageText += item.str;
                        
                        // Add space if the next item doesn't start immediately after this one
                        if ('width' in item && item.width > 0) {
                            pageText += ' ';
                        }
                        
                        currentY = item.transform[5];
                    }
                }
                
                // Clean up the text
                pageText = pageText
                    .replace(/\s+/g, ' ') // Replace multiple whitespace with single space
                    .replace(/\n\s+/g, '\n') // Clean up line starts
                    .trim();
                
                if (pageText.length > 0) {
                    if (numPages > 1) {
                        markdown += `## Page ${pageNum}\n\n`;
                    }
                    markdown += pageText + '\n\n';
                }
            }
            
            // Clean up the final markdown
            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
            
            logger.debug(`Converted PDF to ${markdown.length} characters of markdown`);
            return markdown;
            
        } catch (error) {
            logger.error(`Failed to convert PDF ${filePath}:`, error);
            throw error;
        }
    }

    async convertFileToMarkdown(filePath: string, extension: string, logger: Logger): Promise<string> {
        const ext = extension.toLowerCase();
        if (ext === '.pdf') {
            return this.convertPdfToMarkdown(filePath, logger);
        } else if (ext === '.doc') {
            return this.convertDocToMarkdown(filePath, logger);
        } else if (ext === '.docx') {
            return this.convertDocxToMarkdown(filePath, logger);
        } else if (ext === '.html' || ext === '.htm') {
            const content = fs.readFileSync(filePath, { encoding: 'utf8' });
            const cleanHtml = sanitizeHtml(content, {
                allowedTags: [
                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
                ],
                allowedAttributes: {
                    'a': ['href'],
                    'pre': ['class', 'data-language'],
                    'code': ['class', 'data-language'],
                    'div': ['class'],
                    'span': ['class']
                }
            });
            return this.turndownService.turndown(cleanHtml);
        } else {
            throw new Error(`Unsupported file extension for conversion: ${extension}`);
        }
    }

    private async downloadAndConvertPdfFromUrl(url: string, logger: Logger): Promise<string> {
        logger.debug(`Downloading and converting PDF from URL: ${url}`);
        
        try {
            // Download the PDF file
            const response = await axios.get(url, {
                responseType: 'arraybuffer',
                timeout: 60000, // 60 second timeout
                headers: {
                    'User-Agent': 'Mozilla/5.0 (compatible; doc2vec PDF processor)'
                }
            });
            
            if (response.status !== 200) {
                throw new Error(`Failed to download PDF: HTTP ${response.status}`);
            }
            
            logger.debug(`Downloaded PDF (${response.data.byteLength} bytes)`);
            
            // Dynamic import for PDF.js to handle ES module compatibility
            const pdfjsLib = await import('pdfjs-dist/legacy/build/pdf.mjs');
            
            // Convert ArrayBuffer to Uint8Array
            const pdfData = new Uint8Array(response.data);
            
            // Load the PDF document
            const loadingTask = pdfjsLib.getDocument({
                data: pdfData,
                // Disable worker to avoid issues in Node.js environment
                useWorkerFetch: false,
                isEvalSupported: false,
                useSystemFonts: true
            });
            
            const pdfDocument = await loadingTask.promise;
            const numPages = pdfDocument.numPages;
            
            logger.debug(`PDF has ${numPages} pages`);
            
            // Get the filename from URL for the title
            const urlPath = new URL(url).pathname;
            const filename = path.basename(urlPath, '.pdf') || 'document';
            let markdown = `# ${filename}\n\n`;
            
            // Extract text from each page
            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
                const page = await pdfDocument.getPage(pageNum);
                const textContent = await page.getTextContent();
                
                // Combine text items into a readable format
                let pageText = '';
                let currentY = -1;
                
                for (const item of textContent.items) {
                    if ('str' in item) {
                        // If this is a new line (different Y position), add a line break
                        if (currentY !== -1 && Math.abs(item.transform[5] - currentY) > 5) {
                            pageText += '\n';
                        }
                        
                        pageText += item.str;
                        
                        // Add space if the next item doesn't start immediately after this one
                        if ('width' in item && item.width > 0) {
                            pageText += ' ';
                        }
                        
                        currentY = item.transform[5];
                    }
                }
                
                // Clean up the text
                pageText = pageText
                    .replace(/\s+/g, ' ') // Replace multiple whitespace with single space
                    .replace(/\n\s+/g, '\n') // Clean up line starts
                    .trim();
                
                if (pageText.length > 0) {
                    if (numPages > 1) {
                        markdown += `## Page ${pageNum}\n\n`;
                    }
                    markdown += pageText + '\n\n';
                }
            }
            
            // Clean up the final markdown
            markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
            
            logger.debug(`Converted PDF to ${markdown.length} characters of markdown`);
            return markdown;
            
        } catch (error) {
            logger.error(`Failed to download and convert PDF from ${url}:`, error);
            throw error;
        }
    }

    async processDirectory(
        dirPath: string,
        config: LocalDirectorySourceConfig,
        processFileContent: (filePath: string, content: string) => Promise<void>,
        parentLogger: Logger,
        visitedPaths: Set<string> = new Set()
    ): Promise<void> {
        const logger = parentLogger.child('directory-processor');
        logger.info(`Processing directory: ${dirPath}`);
        
        const recursive = config.recursive !== undefined ? config.recursive : true;
        const includeExtensions = config.include_extensions || ['.md', '.txt', '.html', '.htm', '.pdf'];
        const excludeExtensions = config.exclude_extensions || [];
        const encoding = config.encoding || 'utf8' as BufferEncoding;
        
        try {
            const files = fs.readdirSync(dirPath);
            let processedFiles = 0;
            let skippedFiles = 0;
            
            for (const file of files) {
                const filePath = path.join(dirPath, file);

                // A dangling symlink makes statSync throw ENOENT; skip the
                // entry rather than abandoning the rest of the directory
                let stat: fs.Stats;
                try {
                    stat = fs.statSync(filePath);
                } catch (error) {
                    logger.warn(`Skipping unreadable entry ${filePath}:`, error);
                    skippedFiles++;
                    continue;
                }

                // Skip already visited paths
                if (visitedPaths.has(filePath)) {
                    logger.debug(`Skipping already visited path: ${filePath}`);
                    continue;
                }
                
                visitedPaths.add(filePath);
                
                if (stat.isDirectory()) {
                    if (recursive) {
                        await this.processDirectory(filePath, config, processFileContent, logger, visitedPaths);
                    } else {
                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
                    }
                } else if (stat.isFile()) {
                    const extension = path.extname(file).toLowerCase();
                    
                    // Apply extension filters
                    if (excludeExtensions.includes(extension)) {
                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
                        skippedFiles++;
                        continue;
                    }
                    
                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
                        skippedFiles++;
                        continue;
                    }
                    
                    try {
                        logger.info(`Reading file: ${filePath}`);
                        
                        let content: string;
                        let processedContent: string;
                        
                        const convertibleExtensions = ['.pdf', '.doc', '.docx', '.html', '.htm'];
                        if (convertibleExtensions.includes(extension)) {
                            if (extension === '.html' || extension === '.htm') {
                                // For HTML, check raw file size before converting
                                content = fs.readFileSync(filePath, { encoding: encoding as BufferEncoding });
                                if (content.length > config.max_size) {
                                    logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
                                    skippedFiles++;
                                    continue;
                                }
                            }
                            processedContent = await this.convertFileToMarkdown(filePath, extension, logger);
                        } else {
                            // Handle text-based files
                            content = fs.readFileSync(filePath, { encoding: encoding as BufferEncoding });

                            if (content.length > config.max_size) {
                                logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
                                skippedFiles++;
                                continue;
                            }

                            processedContent = content;
                        }
                        
                        // Check size limit for processed content
                        if (processedContent.length > config.max_size) {
                            logger.warn(`Processed content (${processedContent.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
                            skippedFiles++;
                            continue;
                        }
                        
                        await processFileContent(filePath, processedContent);
                        processedFiles++;
                    } catch (error) {
                        logger.error(`Error processing file ${filePath}:`, error);
                    }
                }
            }
            
            logger.info(`Directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
        } catch (error) {
            logger.error(`Error reading directory ${dirPath}:`, error);
        }
    }

    async processCodeDirectory(
        dirPath: string,
        config: CodeSourceConfig,
        processFileContent: (filePath: string, content: string) => Promise<void>,
        parentLogger: Logger,
        visitedPaths: Set<string> = new Set(),
        options?: {
            allowedFiles?: Set<string>;
            mtimeCutoff?: number;
            trackFiles?: Set<string>;
            // The directory the scan started from. Recursive calls pass it down
            // so `exclude_paths` patterns keep matching against the source root
            // and not against the current subdirectory.
            rootPath?: string;
        }
    ): Promise<CodeScanResult> {
        const logger = parentLogger.child('code-directory-processor');
        logger.info(`Processing code directory: ${dirPath}`);

        const recursive = config.recursive !== undefined ? config.recursive : true;
        const includeExtensions = config.include_extensions || [
            '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
            '.py', '.go', '.rs', '.java', '.kt', '.kts', '.swift',
            '.c', '.cc', '.cpp', '.h', '.hpp', '.cs',
            '.rb', '.php', '.scala', '.sql', '.sh', '.bash', '.zsh',
            '.html', '.css', '.scss', '.sass', '.less',
            '.json', '.yaml', '.yml', '.md'
        ];
        const excludeExtensions = config.exclude_extensions || [];
        const excludePaths = config.exclude_paths || [];
        const encoding = config.encoding || ('utf8' as BufferEncoding);
        const rootPath = options?.rootPath ?? dirPath;
        const childOptions = { ...options, rootPath };

        let maxMtime = 0;

        let incomplete = false;

        try {
            const files = fs.readdirSync(dirPath);
            let processedFiles = 0;
            let skippedFiles = 0;

            const allowedFiles = options?.allowedFiles;
            const mtimeCutoff = options?.mtimeCutoff;
            const trackFiles = options?.trackFiles;

            for (const file of files) {
                const filePath = path.join(dirPath, file);

                // statSync follows symlinks, so a dangling one (common in
                // vendored forks) throws ENOENT. Skipping the entry keeps the
                // rest of the directory — letting it throw used to abandon
                // every sibling after it, and cleanup then deleted their chunks.
                let stat: fs.Stats;
                try {
                    stat = fs.statSync(filePath);
                } catch (error) {
                    logger.warn(`Skipping unreadable entry ${filePath}:`, error);
                    skippedFiles++;
                    continue;
                }

                if (visitedPaths.has(filePath)) {
                    logger.debug(`Skipping already visited path: ${filePath}`);
                    continue;
                }

                visitedPaths.add(filePath);

                // An excluded path never reaches trackFiles or maxMtime, so the
                // caller treats it like a deleted file and removes its chunks
                if (excludePaths.length > 0) {
                    const relativePath = path.relative(rootPath, filePath).replace(/\\/g, '/');
                    const excluded = stat.isDirectory()
                        ? Utils.isDirectoryExcluded(relativePath, excludePaths)
                        : Utils.matchesAnyGlob(relativePath, excludePaths);
                    if (excluded) {
                        logger.debug(`Skipping excluded path: ${relativePath}`);
                        if (!stat.isDirectory()) {
                            skippedFiles++;
                        }
                        continue;
                    }
                }

                if (stat.isDirectory()) {
                    if (recursive) {
                        const childResult = await this.processCodeDirectory(
                            filePath,
                            config,
                            processFileContent,
                            logger,
                            visitedPaths,
                            childOptions
                        );
                        processedFiles += childResult.processedFiles;
                        skippedFiles += childResult.skippedFiles;
                        maxMtime = Math.max(maxMtime, childResult.maxMtime);
                        incomplete = incomplete || childResult.incomplete;
                    } else {
                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
                    }
                } else if (stat.isFile()) {
                    const extension = path.extname(file).toLowerCase();

                    if (excludeExtensions.includes(extension)) {
                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
                        skippedFiles++;
                        continue;
                    }

                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
                        skippedFiles++;
                        continue;
                    }

                    trackFiles?.add(filePath);
                    maxMtime = Math.max(maxMtime, stat.mtimeMs);

                    if (allowedFiles && !allowedFiles.has(filePath)) {
                        skippedFiles++;
                        continue;
                    }

                    if (mtimeCutoff !== undefined && stat.mtimeMs <= mtimeCutoff) {
                        skippedFiles++;
                        continue;
                    }

                    try {
                        logger.info(`Reading file: ${filePath}`);
                        const content = fs.readFileSync(filePath, { encoding: encoding as BufferEncoding });

                        if (content.length > config.max_size) {
                            logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
                            skippedFiles++;
                            continue;
                        }

                        await processFileContent(filePath, content);
                        processedFiles++;
                    } catch (error) {
                        logger.error(`Error processing file ${filePath}:`, error);
                    }
                }
            }

            logger.info(`Code directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
            return { processedFiles, skippedFiles, maxMtime, incomplete };
        } catch (error) {
            // The directory could not be listed at all — report the scan as
            // incomplete so the caller skips cleanup instead of treating every
            // file under this path as deleted
            logger.error(`Error reading code directory ${dirPath}:`, error);
            return { processedFiles: 0, skippedFiles: 0, maxMtime, incomplete: true };
        }
    }

    private async getTokenChunker(chunkSize: number | undefined): Promise<TokenChunker> {
        const cacheKey = `${chunkSize || 'default'}`;
        const cached = this.tokenChunkerCache.get(cacheKey);
        if (cached) {
            return cached;
        }

        const chunkerPromise = (async () => {
            const { TokenChunker } = await this.importChonkieModule('@chonkiejs/core');
            return await TokenChunker.create({ chunkSize, tokenizer: 'character' });
        })();

        this.tokenChunkerCache.set(cacheKey, chunkerPromise);
        return chunkerPromise;
    }

    private async getCodeChunker(lang: string, chunkSize: number | undefined): Promise<CodeChunker> {
        const cacheKey = `${lang}:${chunkSize || 'default'}`;
        const cached = this.codeChunkerCache.get(cacheKey);
        if (cached) {
            return cached;
        }

        const chunkerPromise = (async () => {
            const tokenizer = await this.getTokenizer();
            return await CodeChunker.create({
                lang,
                chunkSize,
                tokenCounter: async (text: string) => tokenizer.countTokens(text)
            });
        })();

        this.codeChunkerCache.set(cacheKey, chunkerPromise);
        return chunkerPromise;
    }

    private async getTokenizer(): Promise<Tokenizer> {
        if (!this.tokenizerCache) {
            this.tokenizerCache = (async () => {
                const { Tokenizer } = await this.importChonkieModule('@chonkiejs/core');
                return await Tokenizer.create('character');
            })();
        }

        return this.tokenizerCache;
    }

    private detectCodeLanguage(filePath: string): string | undefined {
        const extension = path.extname(filePath).toLowerCase();
        const languageMap: Record<string, string> = {
            '.ts': 'typescript',
            '.tsx': 'typescript',
            '.js': 'javascript',
            '.jsx': 'javascript',
            '.mjs': 'javascript',
            '.cjs': 'javascript',
            '.py': 'python',
            '.go': 'go',
            '.rs': 'rust',
            '.java': 'java',
            '.kt': 'kotlin',
            '.kts': 'kotlin',
            '.swift': 'swift',
            '.c': 'c',
            '.cc': 'cpp',
            '.cpp': 'cpp',
            '.h': 'cpp',
            '.hpp': 'cpp',
            '.cs': 'csharp',
            '.rb': 'ruby',
            '.php': 'php',
            '.scala': 'scala',
            '.sql': 'sql',
            '.sh': 'bash',
            '.bash': 'bash',
            '.zsh': 'bash',
            '.html': 'html',
            '.css': 'css',
            '.scss': 'scss',
            '.sass': 'scss',
            '.less': 'css',
            '.json': 'json',
            '.yaml': 'yaml',
            '.yml': 'yaml',
            '.md': 'markdown'
        };

        return languageMap[extension];
    }

    private async importChonkieModule(specifier: string): Promise<any> {
        // Use dynamic import() directly - preserved by TypeScript with target ES2020+
        // even in CommonJS mode, as Node.js supports import() in CJS contexts
        return import(specifier);
    }

    async chunkCode(
        code: string,
        sourceConfig: CodeSourceConfig,
        url: string,
        filePath: string,
        branch?: string,
        repo?: string
    ): Promise<DocumentChunk[]> {
        const logger = this.logger.child('code-chunker');
        const normalizedPath = filePath.replace(/\\/g, '/');
        const lang = this.detectCodeLanguage(filePath);
        let chunks: Array<{ text: string }>;

        if (lang === 'markdown') {
            const markdownChunks = await this.chunkMarkdown(code, sourceConfig, url);

            for (const chunk of markdownChunks) {
                if (normalizedPath) {
                    const filePrefix = `[File: ${normalizedPath}]\n`;
                    const searchableText = filePrefix + chunk.content;
                    const chunkId = Utils.generateHash(`${url}::${searchableText}`);

                    chunk.content = searchableText;
                    chunk.metadata.heading_hierarchy = [normalizedPath, ...chunk.metadata.heading_hierarchy.filter(Boolean)];
                    chunk.metadata.section = normalizedPath;
                    chunk.metadata.chunk_id = chunkId;
                    chunk.metadata.hash = chunkId;
                }

                if (branch) {
                    chunk.metadata.branch = branch;
                }

                if (repo) {
                    chunk.metadata.repo = repo;
                }
            }

            logger.debug(`Chunked ${normalizedPath || url}: ${markdownChunks.length} chunks created.`);
            return markdownChunks;
        }

        if (lang) {
            try {
                const codeChunker = await this.getCodeChunker(lang, sourceConfig.chunk_size);
                chunks = await Promise.race([
                    codeChunker.chunk(code),
                    new Promise<never>((_, reject) =>
                        setTimeout(() => reject(new Error(`CodeChunker timed out after 30s for ${normalizedPath || url}`)), 30000)
                    )
                ]);
            } catch (error) {
                logger.warn(`CodeChunker failed for ${normalizedPath || url}, falling back to token chunking:`, error);
                const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
                chunks = await chunker.chunk(code);
            }
        } else {
            const chunker = await this.getTokenChunker(sourceConfig.chunk_size);
            chunks = await chunker.chunk(code);
        }

        const documentChunks: DocumentChunk[] = [];
        let chunkCounter = 0;
        const headingHierarchy = normalizedPath ? [normalizedPath] : [];
        const contextPrefix = normalizedPath ? `[File: ${normalizedPath}]\n` : '';

        for (const chunk of chunks) {
            const content = chunk.text?.trim();
            if (!content) {
                continue;
            }

            // See chunkMarkdown: an unpaired surrogate makes the store's JSON
            // body invalid, so sanitize before the id is derived from the text.
            const searchableText = Utils.stripLoneSurrogates(contextPrefix + content);
            const chunkId = Utils.generateHash(`${url}::${searchableText}`);

            documentChunks.push({
                content: searchableText,
                metadata: {
                    product_name: sourceConfig.product_name,
                    version: sourceConfig.version,
                    ...(branch ? { branch } : {}),
                    ...(repo ? { repo } : {}),
                    heading_hierarchy: headingHierarchy,
                    section: normalizedPath || 'Code',
                    chunk_id: chunkId,
                    url: url,
                    hash: chunkId,
                    chunk_index: chunkCounter,
                    total_chunks: 0
                }
            });

            chunkCounter++;
        }

        const totalChunks = documentChunks.length;
        for (const chunk of documentChunks) {
            chunk.metadata.total_chunks = totalChunks;
        }

        logger.debug(`Chunked ${normalizedPath || url}: ${documentChunks.length} chunks created.`);
        return documentChunks;
    }

    async chunkMarkdown(markdown: string, sourceConfig: SourceConfig, url: string): Promise<DocumentChunk[]> {
        const logger = this.logger.child('chunker');
        
        // --- Configuration (character-based, ~4 chars ≈ 1 token) ---
        const MAX_CHARS = 4000;
        const MIN_CHARS = 600;       // 💡 Merges short sections into the next block
        const OVERLAP_PERCENT = 0.1; // 10% overlap for large splits
        
        const chunks: DocumentChunk[] = [];
        const lines = markdown.split("\n");
        
        let buffer = ""; 
        let headingHierarchy: string[] = [];
        let bufferHeadings: Array<{ level: number; text: string }> = []; // Track headings in current buffer
        let chunkCounter = 0; // Tracks chunk position within this page for ordering
    
        /**
         * Computes the topic hierarchy for merged content.
         * When merging sibling sections (same level), uses their parent heading.
         * Otherwise uses the current hierarchy.
         */
        const computeTopicHierarchy = (): string[] => {
            if (bufferHeadings.length === 0) {
                return headingHierarchy;
            }
            
            // Find the deepest level (most recent headings)
            const deepestLevel = Math.max(...bufferHeadings.map(h => h.level));
            
            // Get all headings at the deepest level
            const deepestHeadings = bufferHeadings.filter(h => h.level === deepestLevel);
            
            // If we have multiple sibling headings at the deepest level, use their parent
            if (deepestHeadings.length > 1 && deepestLevel > 1) {
                // Use parent heading (one level up from the sibling headings)
                // headingHierarchy still contains the parent at index (deepestLevel - 2)
                // We want everything up to (but not including) the deepest level
                return headingHierarchy.slice(0, deepestLevel - 1);
            }
            
            // Single heading or different levels: use the current hierarchy
            // This reflects the most recent heading which is appropriate
            return headingHierarchy;
        };
    
        /**
         * Internal helper to create the final chunk object with injected context.
         */
        const createDocumentChunk = (content: string, hierarchy: string[]): DocumentChunk => {
            // 💡 BREADCRUMB INJECTION
            // We prepend the hierarchy to the text. This makes the vector highly relevant 
            // to searches for parent topics even if the body doesn't mention them.
            const breadcrumbs = hierarchy.filter(h => h).join(" > ");
            const contextPrefix = breadcrumbs ? `[Topic: ${breadcrumbs}]\n` : "";
            // Sanitize before hashing so the id and the stored text always agree:
            // an unpaired surrogate (from a split pair, or straight from the
            // source) makes the JSON body invalid and the write fails.
            const searchableText = Utils.stripLoneSurrogates(contextPrefix + content.trim());
            const chunkId = Utils.generateHash(searchableText);
            
            const chunk: DocumentChunk = {
                content: searchableText,
                metadata: {
                    product_name: sourceConfig.product_name,
                    version: sourceConfig.version,
                    heading_hierarchy: hierarchy.filter(h => h),
                    section: hierarchy[hierarchy.length - 1] || "Introduction",
                    chunk_id: chunkId,
                    url: url,
                    hash: chunkId,
                    chunk_index: Math.floor(chunkCounter),
                    total_chunks: 0  // Placeholder, will be updated after all chunks are created
                }
            };
            chunkCounter++; // Increment for next chunk
            return chunk;
        };
    
        /**
         * Flushes the current buffer into the chunks array.
         * Uses sub-splitting logic if the buffer exceeds MAX_CHARS.
         */
        const flushBuffer = (force = false) => {
            const trimmedBuffer = buffer.trim();
            if (!trimmedBuffer) return;

            const charCount = trimmedBuffer.length;

            // 💡 SEMANTIC MERGING
            // If the current section is too short (like just a title or a one-liner),
            // we don't flush yet unless it's the end of the file (force=true).
            if (charCount < MIN_CHARS && !force) {
                return;
            }

            // Compute the appropriate topic hierarchy for merged content
            const topicHierarchy = computeTopicHierarchy();

            if (charCount > MAX_CHARS) {
                // 💡 OVERLAP SPLITTING
                // If the section is a massive guide, split it but keep headers on every sub-piece.
                const overlapSize = Math.floor(MAX_CHARS * OVERLAP_PERCENT);

                for (let i = 0; i < trimmedBuffer.length; i += (MAX_CHARS - overlapSize)) {
                    // Slice on code points, not code units: a boundary falling
                    // inside a surrogate pair (emoji, some CJK) would leave a
                    // lone surrogate, which Qdrant's JSON parser rejects — the
                    // chunk then fails to store and is silently lost.
                    const subContent = Utils.sliceSafe(trimmedBuffer, i, i + MAX_CHARS);
                    chunks.push(createDocumentChunk(subContent, topicHierarchy));
                }
            } else {
                chunks.push(createDocumentChunk(trimmedBuffer, topicHierarchy));
            }

            buffer = ""; // Reset buffer after successful flush
            bufferHeadings = []; // Reset tracked headings
        };
    
        // --- Main Processing Loop ---
        for (const line of lines) {
            const isHeading = line.startsWith("#");
    
            if (isHeading) {
                // Update Hierarchy Stack for the new heading
                const levelMatch = line.match(/^(#+)/);
                const level = levelMatch ? levelMatch[1].length : 1;
                // Clean heading: remove markdown prefix and anchor links like [](#anchor-id)
                const headingText = line
                    .replace(/^#+\s*/, "")           // Remove ## prefix
                    .replace(/\[.*?\]\(#[^)]*\)/g, "") // Remove [text](#anchor) patterns
                    .replace(/\[\]\(#[^)]*\)/g, "")    // Remove [](#anchor) patterns  
                    .trim();
                
                // Check if we should merge with previous content
                const currentCharCount = buffer.trim().length;
                const hasBufferContent = currentCharCount > 0;
                const bufferIsSmall = currentCharCount < MIN_CHARS;
                
                // Only merge if:
                // 1. Buffer has content and is small
                // 2. Buffer has tracked headings (we're merging sections, not just content)
                // 3. New heading is at same or deeper level than the deepest heading in buffer (siblings or children)
                //    If new heading is shallower (e.g., H2 after H3), it's a new section - flush first
                const deepestBufferLevel = bufferHeadings.length > 0 
                    ? Math.max(...bufferHeadings.map(h => h.level)) 
                    : 0;
                const shouldMerge = hasBufferContent && bufferIsSmall && bufferHeadings.length > 0 &&
                    level >= deepestBufferLevel;
                
                if (!shouldMerge && hasBufferContent) {
                    // Buffer is large enough OR new heading starts a new section - flush first
                    flushBuffer();
                }
                // If shouldMerge is true, we keep the buffer and merge the sections
    
                // Reset hierarchy below this level (e.g., H2 reset should clear previous H3s)
                headingHierarchy = headingHierarchy.slice(0, level - 1);
                headingHierarchy[level - 1] = headingText;
    
                // Track this heading in the buffer
                bufferHeadings.push({ level, text: headingText });
    
                buffer += `${line}\n`;
            } else {
                buffer += `${line}\n`;
                
                // Safety valve: if a single section is huge, flush it periodically
                if (buffer.length >= MAX_CHARS) {
                    flushBuffer();
                }
            }
        }
    
        // Final sweep
        flushBuffer(true); 
        
        // Update all chunks with the final total count
        const totalChunks = Math.floor(chunks.length);
        for (const chunk of chunks) {
            chunk.metadata.total_chunks = totalChunks;
        }
        
        logger.debug(`Chunking complete: ${chunks.length} rich context chunks created.`);
        return chunks;
    }
}