docs: vector-export examples for Milvus, Pinecone, Qdrant

examples/vector-export/README.md

@@ -0,0 +1,70 @@
+# Vector DB export
+
+Crawl a site with Anakin, embed the content, and upsert it into a vector
+database — for building a RAG corpus from scraped web data. One shared
+pipeline (`lib/`), three thin per-database adapters.
+
+```
+lib/chunk.mjs             paragraph-aware text chunking (pure function, tested)
+lib/embed.mjs              OpenAI embeddings call (swap for any provider)
+lib/deterministic-id.mjs   stable IDs so re-running an export upserts, not duplicates
+lib/prepare.mjs            crawlAndChunk() — the Anakin SDK crawl + chunk step, shared by all three
+pinecone.mjs               crawlAndChunk → embed → Pinecone upsert
+qdrant.mjs                 crawlAndChunk → embed → Qdrant upsert
+milvus.mjs                 crawlAndChunk → embed → Milvus / Zilliz Cloud upsert
+```
+
+Only the upsert call differs per database — the crawl, chunk, and embed
+steps are written once in `lib/` and shared.
+
+## Install
+
+```
+npm install @anakin-io/sdk
+```
+
+No vector-DB SDK dependency for any of the three — each adapter talks to
+its database's REST API directly via `fetch`, so there's nothing extra to
+install per backend.
+
+## Run
+
+```
+ANAKIN_API_KEY=ak-... OPENAI_API_KEY=sk-... \
+PINECONE_API_KEY=... PINECONE_INDEX_HOST=... \
+node pinecone.mjs https://docs.example.com
+```
+
+See the header comment in `pinecone.mjs` / `qdrant.mjs` / `milvus.mjs` for
+that database's required env vars and the one-time index/collection setup
+call it expects to already exist.
+
+## Why chunk IDs are deterministic, not random
+
+Every adapter derives each chunk's vector-DB ID from its natural key
+(`${page.url}#${chunkIndex}`) rather than a random UUID or incrementing
+counter — `deterministic-id.mjs` hashes that key into a stable UUID (Qdrant)
+or int64 (Milvus); Pinecone accepts the string key directly. Re-running an
+export against a page that's already indexed overwrites the same rows
+instead of accumulating duplicates every run.
+
+## Tests
+
+The chunking and ID logic are pure functions with real unit tests, run with
+Node's built-in test runner (no extra dev dependency):
+
+```
+node --test examples/vector-export/lib/*.test.mjs
+```
+
+The embed/upsert HTTP calls aren't unit tested here — they need live
+credentials for OpenAI and the target vector DB, so they're exercised
+end-to-end by actually running an adapter script rather than mocked.
+
+## Swapping the embedding provider
+
+`lib/embed.mjs` is one function, `embedTexts(texts, options) -> number[][]`.
+Replace its body with a call to any other embeddings API and every adapter
+keeps working unchanged — they only depend on that return shape. Note the
+dimension must then match what you created the index/collection with
+(1536 for `text-embedding-3-small`, the default here).

examples/vector-export/lib/chunk.mjs

@@ -0,0 +1,57 @@
+/**
+ * Split text into overlapping chunks on paragraph boundaries where possible.
+ *
+ * Shared by every vector-export adapter (Pinecone, Qdrant, Milvus) — the
+ * chunking and embedding steps are identical across all three; only the
+ * upsert call differs per database.
+ *
+ * @param {string} text
+ * @param {{ maxChars?: number, overlapChars?: number }} [options]
+ * @returns {string[]}
+ */
+export function chunkText(text, options = {}) {
+  const maxChars = options.maxChars ?? 2000
+  const overlapChars = options.overlapChars ?? 200
+
+  if (maxChars <= 0) throw new Error('maxChars must be positive')
+  if (overlapChars < 0 || overlapChars >= maxChars) {
+    throw new Error('overlapChars must be >= 0 and less than maxChars')
+  }
+
+  const trimmed = text.trim()
+  if (trimmed.length === 0) return []
+  if (trimmed.length <= maxChars) return [trimmed]
+
+  const paragraphs = trimmed.split(/\n{2,}/).filter((p) => p.trim().length > 0)
+
+  const chunks = []
+  let current = ''
+
+  const flush = () => {
+    if (current.trim().length > 0) chunks.push(current.trim())
+  }
+
+  for (const paragraph of paragraphs) {
+    // A single paragraph longer than maxChars gets hard-split on its own.
+    if (paragraph.length > maxChars) {
+      flush()
+      current = ''
+      for (let i = 0; i < paragraph.length; i += maxChars - overlapChars) {
+        chunks.push(paragraph.slice(i, i + maxChars).trim())
+      }
+      continue
+    }
+
+    const candidate = current ? `${current}\n\n${paragraph}` : paragraph
+    if (candidate.length > maxChars) {
+      flush()
+      const overlapTail = current.slice(-overlapChars)
+      current = overlapTail ? `${overlapTail}\n\n${paragraph}` : paragraph
+    } else {
+      current = candidate
+    }
+  }
+  flush()
+
+  return chunks
+}

examples/vector-export/lib/chunk.test.mjs

@@ -0,0 +1,48 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { chunkText } from './chunk.mjs'
+
+test('returns empty array for empty/whitespace input', () => {
+  assert.deepEqual(chunkText(''), [])
+  assert.deepEqual(chunkText('   \n  '), [])
+})
+
+test('returns a single chunk when text fits under maxChars', () => {
+  const text = 'short paragraph'
+  assert.deepEqual(chunkText(text, { maxChars: 2000 }), [text])
+})
+
+test('splits on paragraph boundaries when text exceeds maxChars', () => {
+  const a = 'a'.repeat(50)
+  const b = 'b'.repeat(50)
+  const c = 'c'.repeat(50)
+  const text = [a, b, c].join('\n\n')
+
+  const chunks = chunkText(text, { maxChars: 110, overlapChars: 10 })
+  assert.ok(chunks.length >= 2, 'should split into multiple chunks')
+  for (const chunk of chunks) {
+    assert.ok(chunk.length <= 110, `chunk exceeds maxChars: ${chunk.length}`)
+  }
+  // every source paragraph's content must survive somewhere in the output
+  for (const part of [a, b, c]) {
+    assert.ok(
+      chunks.some((chunk) => chunk.includes(part)),
+      `lost content: ${part.slice(0, 10)}...`,
+    )
+  }
+})
+
+test('hard-splits a single paragraph longer than maxChars', () => {
+  const longParagraph = 'x'.repeat(500)
+  const chunks = chunkText(longParagraph, { maxChars: 100, overlapChars: 20 })
+  assert.ok(chunks.length > 1)
+  for (const chunk of chunks) {
+    assert.ok(chunk.length <= 100)
+  }
+})
+
+test('rejects invalid options', () => {
+  assert.throws(() => chunkText('hello world', { maxChars: 0 }))
+  assert.throws(() => chunkText('hello world', { maxChars: 100, overlapChars: 100 }))
+  assert.throws(() => chunkText('hello world', { maxChars: 100, overlapChars: -1 }))
+})

examples/vector-export/lib/deterministic-id.mjs

@@ -0,0 +1,42 @@
+import { createHash } from 'node:crypto'
+
+/**
+ * Derive a stable UUID (v4 format, but deterministic) from an arbitrary
+ * string. Used for vector DBs (Qdrant) that require a UUID or integer point
+ * ID rather than an arbitrary string — hashing the source chunk's natural
+ * key (`${url}#${chunkIndex}`) means re-running an export against the same
+ * page overwrites the same point instead of creating a duplicate.
+ *
+ * @param {string} input
+ * @returns {string}
+ */
+export function deterministicUuid(input) {
+  const hash = createHash('sha1').update(input).digest('hex')
+  return [
+    hash.slice(0, 8),
+    hash.slice(8, 12),
+    // Force version nibble to '4' for a syntactically valid v4-shaped UUID —
+    // this is a stable identifier, not a cryptographically random one.
+    `4${hash.slice(13, 16)}`,
+    hash.slice(16, 20),
+    hash.slice(20, 32),
+  ].join('-')
+}
+
+/**
+ * Derive a stable positive int64-range integer from an arbitrary string —
+ * for vector DBs (Milvus) whose default primary key type is Int64 rather
+ * than a UUID.
+ *
+ * @param {string} input
+ * @returns {string} decimal string, since JS numbers lose precision above
+ *   2^53 and Milvus's Int64 range exceeds that — callers should send this
+ *   as a string in the request body, which Milvus's REST API accepts for
+ *   Int64 fields.
+ */
+export function deterministicInt64(input) {
+  const hash = createHash('sha1').update(input).digest('hex')
+  // Top bit must stay 0 to keep this a positive signed int64.
+  const masked = BigInt('0x' + hash.slice(0, 16)) & 0x7fffffffffffffffn
+  return masked.toString()
+}

examples/vector-export/lib/deterministic-id.test.mjs

@@ -0,0 +1,38 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { deterministicUuid, deterministicInt64 } from './deterministic-id.mjs'
+
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/
+
+test('produces a syntactically valid UUID', () => {
+  assert.match(deterministicUuid('https://example.com/page#0'), UUID_PATTERN)
+})
+
+test('is deterministic for the same input', () => {
+  const input = 'https://example.com/page#3'
+  assert.equal(deterministicUuid(input), deterministicUuid(input))
+})
+
+test('differs for different input', () => {
+  assert.notEqual(
+    deterministicUuid('https://example.com/page#0'),
+    deterministicUuid('https://example.com/page#1'),
+  )
+})
+
+test('deterministicInt64 produces a positive integer string within int64 range', () => {
+  const value = deterministicInt64('https://example.com/page#0')
+  assert.match(value, /^\d+$/)
+  const asBigInt = BigInt(value)
+  assert.ok(asBigInt > 0n)
+  assert.ok(asBigInt <= 0x7fffffffffffffffn)
+})
+
+test('deterministicInt64 is deterministic and differs across input', () => {
+  const input = 'https://example.com/page#3'
+  assert.equal(deterministicInt64(input), deterministicInt64(input))
+  assert.notEqual(
+    deterministicInt64('https://example.com/page#0'),
+    deterministicInt64('https://example.com/page#1'),
+  )
+})

examples/vector-export/lib/embed.mjs

@@ -0,0 +1,40 @@
+/**
+ * Embed text chunks via OpenAI's embeddings API.
+ *
+ * Any embedding provider works here — this uses OpenAI because it's the
+ * most universally available default, not because it's required. Swap this
+ * file for a different provider's REST call and every adapter (Pinecone,
+ * Qdrant, Milvus) keeps working unchanged, since they only depend on
+ * `embedTexts`'s return shape (an array of number[] in the same order as
+ * the input texts).
+ *
+ * @param {string[]} texts
+ * @param {{ apiKey: string, model?: string }} options
+ * @returns {Promise<number[][]>}
+ */
+export async function embedTexts(texts, options) {
+  const { apiKey, model = 'text-embedding-3-small' } = options
+  if (!apiKey) throw new Error('embedTexts: apiKey is required')
+  if (texts.length === 0) return []
+
+  const res = await fetch('https://api.openai.com/v1/embeddings', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ model, input: texts }),
+  })
+
+  if (!res.ok) {
+    const body = await res.text()
+    throw new Error(`OpenAI embeddings request failed (${res.status}): ${body}`)
+  }
+
+  const json = await res.json()
+  // OpenAI returns embeddings in the same order as the input array, each
+  // tagged with its `index` — sort defensively rather than trust order.
+  return json.data
+    .sort((a, b) => a.index - b.index)
+    .map((item) => item.embedding)
+}

examples/vector-export/lib/prepare.mjs

@@ -0,0 +1,35 @@
+import { Anakin } from '@anakin-io/sdk'
+import { chunkText } from './chunk.mjs'
+
+/**
+ * Crawl a site with the Anakin SDK and chunk every page's markdown.
+ *
+ * Shared by every adapter — only the upsert call differs per vector DB.
+ *
+ * @param {string} url
+ * @param {{ anakinApiKey: string, maxPages?: number, maxChars?: number, overlapChars?: number }} options
+ * @returns {Promise<{ id: string, text: string, sourceUrl: string }[]>}
+ */
+export async function crawlAndChunk(url, options) {
+  const { anakinApiKey, maxPages = 20, maxChars, overlapChars } = options
+  const client = new Anakin({ apiKey: anakinApiKey })
+
+  const result = await client.crawl(url, {
+    formats: ['markdown'],
+    maxPages,
+  })
+
+  const records = []
+  for (const page of result.pages) {
+    if (page.status !== 'completed' || !page.markdown) continue
+    const chunks = chunkText(page.markdown, { maxChars, overlapChars })
+    chunks.forEach((text, i) => {
+      records.push({
+        id: `${page.url}#${i}`,
+        text,
+        sourceUrl: page.url,
+      })
+    })
+  }
+  return records
+}