From ee17ddf54b5361389e6620c412339ada46dc7e3b Mon Sep 17 00:00:00 2001
From: Anakin Engineering <engineering@anakin.io>
Date: Tue, 28 Jul 2026 17:16:47 +0530
Subject: [PATCH] docs: add vector-export examples for Milvus, Pinecone, Qdrant

Crawl a site with Anakin, chunk + embed the content, and upsert it
into a vector DB. Each example shares a lib/ (crawlAndChunk, embed,
deterministic-id) and differs only in the DB-specific upsert call.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 examples/vector-export/README.md              | 70 ++++++++++++++
 examples/vector-export/lib/chunk.mjs          | 57 +++++++++++
 examples/vector-export/lib/chunk.test.mjs     | 48 ++++++++++
 .../vector-export/lib/deterministic-id.mjs    | 42 ++++++++
 .../lib/deterministic-id.test.mjs             | 38 ++++++++
 examples/vector-export/lib/embed.mjs          | 40 ++++++++
 examples/vector-export/lib/prepare.mjs        | 35 +++++++
 examples/vector-export/milvus.mjs             | 95 +++++++++++++++++++
 examples/vector-export/pinecone.mjs           | 89 +++++++++++++++++
 examples/vector-export/qdrant.mjs             | 88 +++++++++++++++++
 10 files changed, 602 insertions(+)
 create mode 100644 examples/vector-export/README.md
 create mode 100644 examples/vector-export/lib/chunk.mjs
 create mode 100644 examples/vector-export/lib/chunk.test.mjs
 create mode 100644 examples/vector-export/lib/deterministic-id.mjs
 create mode 100644 examples/vector-export/lib/deterministic-id.test.mjs
 create mode 100644 examples/vector-export/lib/embed.mjs
 create mode 100644 examples/vector-export/lib/prepare.mjs
 create mode 100644 examples/vector-export/milvus.mjs
 create mode 100644 examples/vector-export/pinecone.mjs
 create mode 100644 examples/vector-export/qdrant.mjs

diff --git a/examples/vector-export/README.md b/examples/vector-export/README.md
new file mode 100644
index 0000000..ff78079
--- /dev/null
+++ b/examples/vector-export/README.md
@@ -0,0 +1,70 @@
+# Vector DB export
+
+Crawl a site with Anakin, embed the content, and upsert it into a vector
+database — for building a RAG corpus from scraped web data. One shared
+pipeline (`lib/`), three thin per-database adapters.
+
+```
+lib/chunk.mjs             paragraph-aware text chunking (pure function, tested)
+lib/embed.mjs              OpenAI embeddings call (swap for any provider)
+lib/deterministic-id.mjs   stable IDs so re-running an export upserts, not duplicates
+lib/prepare.mjs            crawlAndChunk() — the Anakin SDK crawl + chunk step, shared by all three
+pinecone.mjs               crawlAndChunk → embed → Pinecone upsert
+qdrant.mjs                 crawlAndChunk → embed → Qdrant upsert
+milvus.mjs                 crawlAndChunk → embed → Milvus / Zilliz Cloud upsert
+```
+
+Only the upsert call differs per database — the crawl, chunk, and embed
+steps are written once in `lib/` and shared.
+
+## Install
+
+```
+npm install @anakin-io/sdk
+```
+
+No vector-DB SDK dependency for any of the three — each adapter talks to
+its database's REST API directly via `fetch`, so there's nothing extra to
+install per backend.
+
+## Run
+
+```
+ANAKIN_API_KEY=ak-... OPENAI_API_KEY=sk-... \
+PINECONE_API_KEY=... PINECONE_INDEX_HOST=... \
+node pinecone.mjs https://docs.example.com
+```
+
+See the header comment in `pinecone.mjs` / `qdrant.mjs` / `milvus.mjs` for
+that database's required env vars and the one-time index/collection setup
+call it expects to already exist.
+
+## Why chunk IDs are deterministic, not random
+
+Every adapter derives each chunk's vector-DB ID from its natural key
+(`${page.url}#${chunkIndex}`) rather than a random UUID or incrementing
+counter — `deterministic-id.mjs` hashes that key into a stable UUID (Qdrant)
+or int64 (Milvus); Pinecone accepts the string key directly. Re-running an
+export against a page that's already indexed overwrites the same rows
+instead of accumulating duplicates every run.
+
+## Tests
+
+The chunking and ID logic are pure functions with real unit tests, run with
+Node's built-in test runner (no extra dev dependency):
+
+```
+node --test examples/vector-export/lib/*.test.mjs
+```
+
+The embed/upsert HTTP calls aren't unit tested here — they need live
+credentials for OpenAI and the target vector DB, so they're exercised
+end-to-end by actually running an adapter script rather than mocked.
+
+## Swapping the embedding provider
+
+`lib/embed.mjs` is one function, `embedTexts(texts, options) -> number[][]`.
+Replace its body with a call to any other embeddings API and every adapter
+keeps working unchanged — they only depend on that return shape. Note the
+dimension must then match what you created the index/collection with
+(1536 for `text-embedding-3-small`, the default here).
diff --git a/examples/vector-export/lib/chunk.mjs b/examples/vector-export/lib/chunk.mjs
new file mode 100644
index 0000000..186fea7
--- /dev/null
+++ b/examples/vector-export/lib/chunk.mjs
@@ -0,0 +1,57 @@
+/**
+ * Split text into overlapping chunks on paragraph boundaries where possible.
+ *
+ * Shared by every vector-export adapter (Pinecone, Qdrant, Milvus) — the
+ * chunking and embedding steps are identical across all three; only the
+ * upsert call differs per database.
+ *
+ * @param {string} text
+ * @param {{ maxChars?: number, overlapChars?: number }} [options]
+ * @returns {string[]}
+ */
+export function chunkText(text, options = {}) {
+  const maxChars = options.maxChars ?? 2000
+  const overlapChars = options.overlapChars ?? 200
+
+  if (maxChars <= 0) throw new Error('maxChars must be positive')
+  if (overlapChars < 0 || overlapChars >= maxChars) {
+    throw new Error('overlapChars must be >= 0 and less than maxChars')
+  }
+
+  const trimmed = text.trim()
+  if (trimmed.length === 0) return []
+  if (trimmed.length <= maxChars) return [trimmed]
+
+  const paragraphs = trimmed.split(/\n{2,}/).filter((p) => p.trim().length > 0)
+
+  const chunks = []
+  let current = ''
+
+  const flush = () => {
+    if (current.trim().length > 0) chunks.push(current.trim())
+  }
+
+  for (const paragraph of paragraphs) {
+    // A single paragraph longer than maxChars gets hard-split on its own.
+    if (paragraph.length > maxChars) {
+      flush()
+      current = ''
+      for (let i = 0; i < paragraph.length; i += maxChars - overlapChars) {
+        chunks.push(paragraph.slice(i, i + maxChars).trim())
+      }
+      continue
+    }
+
+    const candidate = current ? `${current}\n\n${paragraph}` : paragraph
+    if (candidate.length > maxChars) {
+      flush()
+      const overlapTail = current.slice(-overlapChars)
+      current = overlapTail ? `${overlapTail}\n\n${paragraph}` : paragraph
+    } else {
+      current = candidate
+    }
+  }
+  flush()
+
+  return chunks
+}
diff --git a/examples/vector-export/lib/chunk.test.mjs b/examples/vector-export/lib/chunk.test.mjs
new file mode 100644
index 0000000..1f8e0d7
--- /dev/null
+++ b/examples/vector-export/lib/chunk.test.mjs
@@ -0,0 +1,48 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { chunkText } from './chunk.mjs'
+
+test('returns empty array for empty/whitespace input', () => {
+  assert.deepEqual(chunkText(''), [])
+  assert.deepEqual(chunkText('   \n  '), [])
+})
+
+test('returns a single chunk when text fits under maxChars', () => {
+  const text = 'short paragraph'
+  assert.deepEqual(chunkText(text, { maxChars: 2000 }), [text])
+})
+
+test('splits on paragraph boundaries when text exceeds maxChars', () => {
+  const a = 'a'.repeat(50)
+  const b = 'b'.repeat(50)
+  const c = 'c'.repeat(50)
+  const text = [a, b, c].join('\n\n')
+
+  const chunks = chunkText(text, { maxChars: 110, overlapChars: 10 })
+  assert.ok(chunks.length >= 2, 'should split into multiple chunks')
+  for (const chunk of chunks) {
+    assert.ok(chunk.length <= 110, `chunk exceeds maxChars: ${chunk.length}`)
+  }
+  // every source paragraph's content must survive somewhere in the output
+  for (const part of [a, b, c]) {
+    assert.ok(
+      chunks.some((chunk) => chunk.includes(part)),
+      `lost content: ${part.slice(0, 10)}...`,
+    )
+  }
+})
+
+test('hard-splits a single paragraph longer than maxChars', () => {
+  const longParagraph = 'x'.repeat(500)
+  const chunks = chunkText(longParagraph, { maxChars: 100, overlapChars: 20 })
+  assert.ok(chunks.length > 1)
+  for (const chunk of chunks) {
+    assert.ok(chunk.length <= 100)
+  }
+})
+
+test('rejects invalid options', () => {
+  assert.throws(() => chunkText('hello world', { maxChars: 0 }))
+  assert.throws(() => chunkText('hello world', { maxChars: 100, overlapChars: 100 }))
+  assert.throws(() => chunkText('hello world', { maxChars: 100, overlapChars: -1 }))
+})
diff --git a/examples/vector-export/lib/deterministic-id.mjs b/examples/vector-export/lib/deterministic-id.mjs
new file mode 100644
index 0000000..709e7f3
--- /dev/null
+++ b/examples/vector-export/lib/deterministic-id.mjs
@@ -0,0 +1,42 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * Derive a stable UUID (v4 format, but deterministic) from an arbitrary
+ * string. Used for vector DBs (Qdrant) that require a UUID or integer point
+ * ID rather than an arbitrary string — hashing the source chunk's natural
+ * key (`${url}#${chunkIndex}`) means re-running an export against the same
+ * page overwrites the same point instead of creating a duplicate.
+ *
+ * @param {string} input
+ * @returns {string}
+ */
+export function deterministicUuid(input) {
+  const hash = createHash('sha1').update(input).digest('hex')
+  return [
+    hash.slice(0, 8),
+    hash.slice(8, 12),
+    // Force version nibble to '4' for a syntactically valid v4-shaped UUID —
+    // this is a stable identifier, not a cryptographically random one.
+    `4${hash.slice(13, 16)}`,
+    hash.slice(16, 20),
+    hash.slice(20, 32),
+  ].join('-')
+}
+
+/**
+ * Derive a stable positive int64-range integer from an arbitrary string —
+ * for vector DBs (Milvus) whose default primary key type is Int64 rather
+ * than a UUID.
+ *
+ * @param {string} input
+ * @returns {string} decimal string, since JS numbers lose precision above
+ *   2^53 and Milvus's Int64 range exceeds that — callers should send this
+ *   as a string in the request body, which Milvus's REST API accepts for
+ *   Int64 fields.
+ */
+export function deterministicInt64(input) {
+  const hash = createHash('sha1').update(input).digest('hex')
+  // Top bit must stay 0 to keep this a positive signed int64.
+  const masked = BigInt('0x' + hash.slice(0, 16)) & 0x7fffffffffffffffn
+  return masked.toString()
+}
diff --git a/examples/vector-export/lib/deterministic-id.test.mjs b/examples/vector-export/lib/deterministic-id.test.mjs
new file mode 100644
index 0000000..20561ea
--- /dev/null
+++ b/examples/vector-export/lib/deterministic-id.test.mjs
@@ -0,0 +1,38 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { deterministicUuid, deterministicInt64 } from './deterministic-id.mjs'
+
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/
+
+test('produces a syntactically valid UUID', () => {
+  assert.match(deterministicUuid('https://example.com/page#0'), UUID_PATTERN)
+})
+
+test('is deterministic for the same input', () => {
+  const input = 'https://example.com/page#3'
+  assert.equal(deterministicUuid(input), deterministicUuid(input))
+})
+
+test('differs for different input', () => {
+  assert.notEqual(
+    deterministicUuid('https://example.com/page#0'),
+    deterministicUuid('https://example.com/page#1'),
+  )
+})
+
+test('deterministicInt64 produces a positive integer string within int64 range', () => {
+  const value = deterministicInt64('https://example.com/page#0')
+  assert.match(value, /^\d+$/)
+  const asBigInt = BigInt(value)
+  assert.ok(asBigInt > 0n)
+  assert.ok(asBigInt <= 0x7fffffffffffffffn)
+})
+
+test('deterministicInt64 is deterministic and differs across input', () => {
+  const input = 'https://example.com/page#3'
+  assert.equal(deterministicInt64(input), deterministicInt64(input))
+  assert.notEqual(
+    deterministicInt64('https://example.com/page#0'),
+    deterministicInt64('https://example.com/page#1'),
+  )
+})
diff --git a/examples/vector-export/lib/embed.mjs b/examples/vector-export/lib/embed.mjs
new file mode 100644
index 0000000..f1008f2
--- /dev/null
+++ b/examples/vector-export/lib/embed.mjs
@@ -0,0 +1,40 @@
+/**
+ * Embed text chunks via OpenAI's embeddings API.
+ *
+ * Any embedding provider works here — this uses OpenAI because it's the
+ * most universally available default, not because it's required. Swap this
+ * file for a different provider's REST call and every adapter (Pinecone,
+ * Qdrant, Milvus) keeps working unchanged, since they only depend on
+ * `embedTexts`'s return shape (an array of number[] in the same order as
+ * the input texts).
+ *
+ * @param {string[]} texts
+ * @param {{ apiKey: string, model?: string }} options
+ * @returns {Promise<number[][]>}
+ */
+export async function embedTexts(texts, options) {
+  const { apiKey, model = 'text-embedding-3-small' } = options
+  if (!apiKey) throw new Error('embedTexts: apiKey is required')
+  if (texts.length === 0) return []
+
+  const res = await fetch('https://api.openai.com/v1/embeddings', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ model, input: texts }),
+  })
+
+  if (!res.ok) {
+    const body = await res.text()
+    throw new Error(`OpenAI embeddings request failed (${res.status}): ${body}`)
+  }
+
+  const json = await res.json()
+  // OpenAI returns embeddings in the same order as the input array, each
+  // tagged with its `index` — sort defensively rather than trust order.
+  return json.data
+    .sort((a, b) => a.index - b.index)
+    .map((item) => item.embedding)
+}
diff --git a/examples/vector-export/lib/prepare.mjs b/examples/vector-export/lib/prepare.mjs
new file mode 100644
index 0000000..b7706c2
--- /dev/null
+++ b/examples/vector-export/lib/prepare.mjs
@@ -0,0 +1,35 @@
+import { Anakin } from '@anakin-io/sdk'
+import { chunkText } from './chunk.mjs'
+
+/**
+ * Crawl a site with the Anakin SDK and chunk every page's markdown.
+ *
+ * Shared by every adapter — only the upsert call differs per vector DB.
+ *
+ * @param {string} url
+ * @param {{ anakinApiKey: string, maxPages?: number, maxChars?: number, overlapChars?: number }} options
+ * @returns {Promise<{ id: string, text: string, sourceUrl: string }[]>}
+ */
+export async function crawlAndChunk(url, options) {
+  const { anakinApiKey, maxPages = 20, maxChars, overlapChars } = options
+  const client = new Anakin({ apiKey: anakinApiKey })
+
+  const result = await client.crawl(url, {
+    formats: ['markdown'],
+    maxPages,
+  })
+
+  const records = []
+  for (const page of result.pages) {
+    if (page.status !== 'completed' || !page.markdown) continue
+    const chunks = chunkText(page.markdown, { maxChars, overlapChars })
+    chunks.forEach((text, i) => {
+      records.push({
+        id: `${page.url}#${i}`,
+        text,
+        sourceUrl: page.url,
+      })
+    })
+  }
+  return records
+}
diff --git a/examples/vector-export/milvus.mjs b/examples/vector-export/milvus.mjs
new file mode 100644
index 0000000..bd14216
--- /dev/null
+++ b/examples/vector-export/milvus.mjs
@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+/**
+ * Crawl a site with Anakin, embed the content, and upsert it into Milvus
+ * (self-hosted with the v2 HTTP gateway, or Zilliz Cloud).
+ *
+ * Usage:
+ *   ANAKIN_API_KEY=ak-... OPENAI_API_KEY=sk-... \
+ *   MILVUS_URL=https://your-cluster.zillizcloud.com MILVUS_TOKEN=... \
+ *   MILVUS_COLLECTION=chunks \
+ *   node milvus.mjs https://docs.example.com
+ *
+ * Prerequisite: the collection must exist with dimension 1536 (matching
+ * text-embedding-3-small), dynamic field enabled (default when created via
+ * the shorthand below) so `text`/`source_url` don't need declaring upfront:
+ *   POST {MILVUS_URL}/v2/vectordb/collections/create
+ *   { "collectionName": "chunks", "dimension": 1536, "metricType": "COSINE" }
+ *
+ * MILVUS_TOKEN is a Zilliz Cloud API key (`Bearer <key>`) or, for
+ * self-hosted Milvus with RBAC, `Bearer <username>:<password>` — check your
+ * cluster's auth docs, the header shape differs between the two.
+ */
+
+import { crawlAndChunk } from './lib/prepare.mjs'
+import { embedTexts } from './lib/embed.mjs'
+import { deterministicInt64 } from './lib/deterministic-id.mjs'
+
+const BATCH_SIZE = 100
+
+async function upsertBatch(milvusUrl, token, collectionName, data) {
+  const res = await fetch(`${milvusUrl}/v2/vectordb/entities/upsert`, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ collectionName, data }),
+  })
+  if (!res.ok) {
+    throw new Error(`Milvus upsert failed (${res.status}): ${await res.text()}`)
+  }
+}
+
+async function main() {
+  const url = process.argv[2]
+  if (!url) {
+    console.error('Usage: node milvus.mjs <url>')
+    process.exit(1)
+  }
+
+  const anakinApiKey = requireEnv('ANAKIN_API_KEY')
+  const openaiApiKey = requireEnv('OPENAI_API_KEY')
+  const milvusUrl = requireEnv('MILVUS_URL').replace(/\/$/, '')
+  const milvusToken = requireEnv('MILVUS_TOKEN')
+  const collectionName = requireEnv('MILVUS_COLLECTION')
+
+  console.log(`Crawling ${url}...`)
+  const records = await crawlAndChunk(url, { anakinApiKey })
+  console.log(`${records.length} chunks to embed and upsert.`)
+  if (records.length === 0) return
+
+  for (let i = 0; i < records.length; i += BATCH_SIZE) {
+    const batch = records.slice(i, i + BATCH_SIZE)
+    const embeddings = await embedTexts(
+      batch.map((r) => r.text),
+      { apiKey: openaiApiKey },
+    )
+    const data = batch.map((record, j) => ({
+      // Deterministic from the chunk's natural key, same reasoning as the
+      // Qdrant adapter's deterministicUuid — re-running against the same
+      // page overwrites the same rows instead of accumulating duplicates.
+      id: deterministicInt64(record.id),
+      vector: embeddings[j],
+      text: record.text,
+      source_url: record.sourceUrl,
+    }))
+    await upsertBatch(milvusUrl, milvusToken, collectionName, data)
+    console.log(`Upserted ${Math.min(i + BATCH_SIZE, records.length)}/${records.length}`)
+  }
+
+  console.log('Done.')
+}
+
+function requireEnv(name) {
+  const value = process.env[name]
+  if (!value) {
+    console.error(`Missing required env var: ${name}`)
+    process.exit(1)
+  }
+  return value
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})
diff --git a/examples/vector-export/pinecone.mjs b/examples/vector-export/pinecone.mjs
new file mode 100644
index 0000000..30947f3
--- /dev/null
+++ b/examples/vector-export/pinecone.mjs
@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+/**
+ * Crawl a site with Anakin, embed the content, and upsert it into Pinecone.
+ *
+ * Usage:
+ *   ANAKIN_API_KEY=ak-... OPENAI_API_KEY=sk-... \
+ *   PINECONE_API_KEY=... PINECONE_INDEX_HOST=example-index-xxxx.svc.us-east1-gcp.pinecone.io \
+ *   node pinecone.mjs https://docs.example.com
+ *
+ * Prerequisite: a Pinecone index must already exist with dimension 1536
+ * (matching text-embedding-3-small) — create one first via
+ *   POST https://api.pinecone.io/indexes
+ * with header `Api-Key` and body:
+ *   { "name": "...", "dimension": 1536, "metric": "cosine",
+ *     "spec": { "serverless": { "cloud": "aws", "region": "us-east-1" } } }
+ * The response's `host` field is PINECONE_INDEX_HOST below — Pinecone's
+ * data-plane API is per-index, there's no single fixed base URL.
+ */
+
+import { crawlAndChunk } from './lib/prepare.mjs'
+import { embedTexts } from './lib/embed.mjs'
+
+const PINECONE_API_VERSION = '2025-10'
+const BATCH_SIZE = 100 // Pinecone allows up to 1000/request; keep well under.
+
+async function upsertBatch(indexHost, apiKey, vectors, namespace) {
+  const res = await fetch(`https://${indexHost}/vectors/upsert`, {
+    method: 'POST',
+    headers: {
+      'Api-Key': apiKey,
+      'Content-Type': 'application/json',
+      'X-Pinecone-Api-Version': PINECONE_API_VERSION,
+    },
+    body: JSON.stringify({ vectors, namespace }),
+  })
+  if (!res.ok) {
+    throw new Error(`Pinecone upsert failed (${res.status}): ${await res.text()}`)
+  }
+}
+
+async function main() {
+  const url = process.argv[2]
+  if (!url) {
+    console.error('Usage: node pinecone.mjs <url>')
+    process.exit(1)
+  }
+
+  const anakinApiKey = requireEnv('ANAKIN_API_KEY')
+  const openaiApiKey = requireEnv('OPENAI_API_KEY')
+  const pineconeApiKey = requireEnv('PINECONE_API_KEY')
+  const indexHost = requireEnv('PINECONE_INDEX_HOST')
+  const namespace = process.env.PINECONE_NAMESPACE
+
+  console.log(`Crawling ${url}...`)
+  const records = await crawlAndChunk(url, { anakinApiKey })
+  console.log(`${records.length} chunks to embed and upsert.`)
+  if (records.length === 0) return
+
+  for (let i = 0; i < records.length; i += BATCH_SIZE) {
+    const batch = records.slice(i, i + BATCH_SIZE)
+    const embeddings = await embedTexts(
+      batch.map((r) => r.text),
+      { apiKey: openaiApiKey },
+    )
+    const vectors = batch.map((record, j) => ({
+      id: record.id,
+      values: embeddings[j],
+      metadata: { text: record.text, source_url: record.sourceUrl },
+    }))
+    await upsertBatch(indexHost, pineconeApiKey, vectors, namespace)
+    console.log(`Upserted ${Math.min(i + BATCH_SIZE, records.length)}/${records.length}`)
+  }
+
+  console.log('Done.')
+}
+
+function requireEnv(name) {
+  const value = process.env[name]
+  if (!value) {
+    console.error(`Missing required env var: ${name}`)
+    process.exit(1)
+  }
+  return value
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})
diff --git a/examples/vector-export/qdrant.mjs b/examples/vector-export/qdrant.mjs
new file mode 100644
index 0000000..f1ff3d3
--- /dev/null
+++ b/examples/vector-export/qdrant.mjs
@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * Crawl a site with Anakin, embed the content, and upsert it into Qdrant.
+ *
+ * Usage:
+ *   ANAKIN_API_KEY=ak-... OPENAI_API_KEY=sk-... \
+ *   QDRANT_URL=https://xyz.cloud.qdrant.io:6333 QDRANT_API_KEY=... \
+ *   QDRANT_COLLECTION=my-collection \
+ *   node qdrant.mjs https://docs.example.com
+ *
+ * Prerequisite: the collection must exist with vector size 1536 (matching
+ * text-embedding-3-small) — create one first:
+ *   PUT {QDRANT_URL}/collections/{QDRANT_COLLECTION}
+ *   { "vectors": { "size": 1536, "distance": "Cosine" } }
+ */
+
+import { crawlAndChunk } from './lib/prepare.mjs'
+import { embedTexts } from './lib/embed.mjs'
+import { deterministicUuid } from './lib/deterministic-id.mjs'
+
+const BATCH_SIZE = 100
+
+async function upsertBatch(qdrantUrl, apiKey, collection, points) {
+  const res = await fetch(`${qdrantUrl}/collections/${collection}/points`, {
+    method: 'PUT',
+    headers: {
+      ...(apiKey ? { 'api-key': apiKey } : {}),
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ points }),
+  })
+  if (!res.ok) {
+    throw new Error(`Qdrant upsert failed (${res.status}): ${await res.text()}`)
+  }
+}
+
+async function main() {
+  const url = process.argv[2]
+  if (!url) {
+    console.error('Usage: node qdrant.mjs <url>')
+    process.exit(1)
+  }
+
+  const anakinApiKey = requireEnv('ANAKIN_API_KEY')
+  const openaiApiKey = requireEnv('OPENAI_API_KEY')
+  const qdrantUrl = requireEnv('QDRANT_URL').replace(/\/$/, '')
+  const collection = requireEnv('QDRANT_COLLECTION')
+  const qdrantApiKey = process.env.QDRANT_API_KEY // optional for self-hosted, open instances
+
+  console.log(`Crawling ${url}...`)
+  const records = await crawlAndChunk(url, { anakinApiKey })
+  console.log(`${records.length} chunks to embed and upsert.`)
+  if (records.length === 0) return
+
+  for (let i = 0; i < records.length; i += BATCH_SIZE) {
+    const batch = records.slice(i, i + BATCH_SIZE)
+    const embeddings = await embedTexts(
+      batch.map((r) => r.text),
+      { apiKey: openaiApiKey },
+    )
+    const points = batch.map((record, j) => ({
+      // Deterministic from the chunk's natural key (url#index), not a
+      // counter — re-running against the same page overwrites the same
+      // points instead of accumulating duplicates on every run.
+      id: deterministicUuid(record.id),
+      vector: embeddings[j],
+      payload: { text: record.text, source_url: record.sourceUrl },
+    }))
+    await upsertBatch(qdrantUrl, qdrantApiKey, collection, points)
+    console.log(`Upserted ${Math.min(i + BATCH_SIZE, records.length)}/${records.length}`)
+  }
+
+  console.log('Done.')
+}
+
+function requireEnv(name) {
+  const value = process.env[name]
+  if (!value) {
+    console.error(`Missing required env var: ${name}`)
+    process.exit(1)
+  }
+  return value
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})