diff --git a/.github/workflows/sync-benchmarks.yml b/.github/workflows/sync-benchmarks.yml new file mode 100644 index 00000000..8caafd48 --- /dev/null +++ b/.github/workflows/sync-benchmarks.yml @@ -0,0 +1,47 @@ +name: Sync Benchmark Data + +on: + schedule: + # Daily at 07:00 UTC + - cron: '0 7 * * *' + workflow_dispatch: # Allow manual trigger + +permissions: + contents: write + pull-requests: write + +jobs: + sync-benchmarks: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Sync benchmark data from dragonflydb/benchmarking + run: yarn sync-benchmarks + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Open PR if anything changed + uses: peter-evans/create-pull-request@v6 + with: + commit-message: 'Sync benchmark data from dragonflydb/benchmarking' + title: 'Sync benchmark data from dragonflydb/benchmarking' + body: | + Automated update of `## Benchmark` sections on command-reference + pages, regenerated from the latest results in + [dragonflydb/benchmarking](https://github.com/dragonflydb/benchmarking). + + Generated by `.github/workflows/sync-benchmarks.yml` running + `yarn sync-benchmarks`. Review the diff before merging. + branch: sync-benchmarks + delete-branch: true diff --git a/docs/command-reference/generic/del.md b/docs/command-reference/generic/del.md index ea230150..4c074ce5 100644 --- a/docs/command-reference/generic/del.md +++ b/docs/command-reference/generic/del.md @@ -3,6 +3,7 @@ description: "Learn how to use Redis DEL command to delete a key." --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # DEL @@ -33,3 +34,26 @@ OK dragonfly> DEL key1 key2 key3 (integer) 2 ``` + + +## Benchmark + + + diff --git a/docs/command-reference/generic/expire.md b/docs/command-reference/generic/expire.md index 6e32419b..5d647fdf 100644 --- a/docs/command-reference/generic/expire.md +++ b/docs/command-reference/generic/expire.md @@ -3,6 +3,7 @@ description: "Learn Redis EXPIRE command that sets a key's time-to-live in secon --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # EXPIRE @@ -86,6 +87,29 @@ dragonfly> TTL mykey (integer) -1 ``` + +## Benchmark + + + + ## Pattern: Navigation Session Imagine you have a web service and you are interested in the latest N pages diff --git a/docs/command-reference/generic/ttl.md b/docs/command-reference/generic/ttl.md index 0b7ba2db..0ea783cd 100644 --- a/docs/command-reference/generic/ttl.md +++ b/docs/command-reference/generic/ttl.md @@ -3,6 +3,7 @@ description: "Learn the Redis TTL command to get remaining time-to-live of a key --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # TTL @@ -37,3 +38,26 @@ dragonfly> EXPIRE mykey 10 dragonfly> TTL mykey (integer) 10 ``` + + +## Benchmark + + + diff --git a/docs/command-reference/hashes/hget.md b/docs/command-reference/hashes/hget.md index 2c39a864..75b96968 100644 --- a/docs/command-reference/hashes/hget.md +++ b/docs/command-reference/hashes/hget.md @@ -3,6 +3,7 @@ description: "Learn how to use Redis HGET command to retrieve the value of a has --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # HGET @@ -33,3 +34,26 @@ dragonfly> HGET myhash field1 dragonfly> HGET myhash field2 (nil) ``` + + +## Benchmark + + + diff --git a/docs/command-reference/hashes/hset.md b/docs/command-reference/hashes/hset.md index efb0f810..b5973ef5 100644 --- a/docs/command-reference/hashes/hset.md +++ b/docs/command-reference/hashes/hset.md @@ -3,6 +3,7 @@ description: "Learn how to use Redis HSET command to set the value of a hash fie --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # HSET @@ -46,3 +47,26 @@ dragonfly> HGETALL myhash 5) "field3" 6) "World" ``` + + +## Benchmark + + + diff --git a/docs/command-reference/lists/lpop.md b/docs/command-reference/lists/lpop.md index cebe1446..27cf5798 100644 --- a/docs/command-reference/lists/lpop.md +++ b/docs/command-reference/lists/lpop.md @@ -2,6 +2,7 @@ description: Learn how to use the Redis LPOP command for removing and getting the first element in the list. --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # LPOP @@ -45,3 +46,26 @@ dragonfly> LRANGE mylist 0 -1 1) "four" 2) "five" ``` + + +## Benchmark + + + diff --git a/docs/command-reference/lists/lpush.md b/docs/command-reference/lists/lpush.md index bc3ef05e..058aecb3 100644 --- a/docs/command-reference/lists/lpush.md +++ b/docs/command-reference/lists/lpush.md @@ -2,6 +2,7 @@ description: Learn how to use Redis LPUSH command to insert an element at the start of a list. --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # LPUSH @@ -42,3 +43,26 @@ dragonfly> LRANGE mylist 0 -1 1) "hello" 2) "world" ``` + + +## Benchmark + + + diff --git a/docs/command-reference/lists/rpop.md b/docs/command-reference/lists/rpop.md index 2ed34dc5..cb77ff19 100644 --- a/docs/command-reference/lists/rpop.md +++ b/docs/command-reference/lists/rpop.md @@ -2,6 +2,7 @@ description: Discover how to use Redis RPOP command to remove and fetch the last element of a list. --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # RPOP @@ -45,3 +46,26 @@ dragonfly> LRANGE mylist 0 -1 1) "one" 2) "two" ``` + + +## Benchmark + + + diff --git a/docs/command-reference/lists/rpush.md b/docs/command-reference/lists/rpush.md index fa3d31a6..307f80a5 100644 --- a/docs/command-reference/lists/rpush.md +++ b/docs/command-reference/lists/rpush.md @@ -2,6 +2,7 @@ description: Learn how to use Redis RPUSH command for appending a value at the end of a list. --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # RPUSH @@ -42,3 +43,26 @@ dragonfly> LRANGE mylist 0 -1 1) "hello" 2) "world" ``` + + +## Benchmark + + + diff --git a/docs/command-reference/sorted-sets/zadd.md b/docs/command-reference/sorted-sets/zadd.md index df859829..7c646a17 100644 --- a/docs/command-reference/sorted-sets/zadd.md +++ b/docs/command-reference/sorted-sets/zadd.md @@ -3,6 +3,7 @@ description: Learn how to use the Redis ZADD command to add members to sorted se --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # ZADD @@ -133,6 +134,29 @@ dragonfly$> ZADD myzset LT CH 20 "player1" (integer) 1 # "player1" was updated because 20 is less than 25. ``` + +## Benchmark + + + + ## Best Practices - Use the `NX` flag if you want to ensure no updates are made to existing members, and members are only added if they do not already exist. diff --git a/docs/command-reference/sorted-sets/zscore.md b/docs/command-reference/sorted-sets/zscore.md index b6aab707..aae2c5ed 100644 --- a/docs/command-reference/sorted-sets/zscore.md +++ b/docs/command-reference/sorted-sets/zscore.md @@ -3,6 +3,7 @@ description: Learn how to use Redis ZSCORE command to get the score associated w --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # ZSCORE @@ -71,6 +72,29 @@ dragonfly$> ZSCORE myzset "memberB" "3.7" ``` + +## Benchmark + + + + ## Best Practices - Use the `ZSCORE` command when you only need to retrieve the score of a single member as it provides a direct and efficient method to perform this query. diff --git a/docs/command-reference/strings/get.md b/docs/command-reference/strings/get.md index 50f807f8..3eaca4be 100644 --- a/docs/command-reference/strings/get.md +++ b/docs/command-reference/strings/get.md @@ -3,6 +3,7 @@ description: Discover how to use Redis GET for fetching the value of a defined k --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # GET @@ -95,6 +96,29 @@ dragonfly$> GET binary_data "\x00\x01\x02\x03" ``` + +## Benchmark + + + + ## Best Practices - Use the `GET` command in combination with `SET` to implement caching mechanisms and session storage efficiently. diff --git a/docs/command-reference/strings/incr.md b/docs/command-reference/strings/incr.md index 960b3276..389aed4f 100644 --- a/docs/command-reference/strings/incr.md +++ b/docs/command-reference/strings/incr.md @@ -3,6 +3,7 @@ description: Learn how to use Redis INCR command for incrementing the integer va --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # INCR @@ -79,6 +80,29 @@ dragonfly$> INCR request_count (integer) 102 ``` + +## Benchmark + + + + ## Best Practices - Use `INCR` to implement performance-efficient counters due to its atomic nature. diff --git a/docs/command-reference/strings/mget.md b/docs/command-reference/strings/mget.md index 9f879d74..3401500c 100644 --- a/docs/command-reference/strings/mget.md +++ b/docs/command-reference/strings/mget.md @@ -3,6 +3,7 @@ description: Learn how to use Redis MGET to retrieve the values of all specified --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # MGET @@ -99,6 +100,29 @@ dragonfly$> MGET key_expiring key_persistent 2) "persistent_value" ``` + +## Benchmark + + + + ## Best Practices - Use `MGET` to minimize network round trips when retrieving values from multiple keys at once. diff --git a/docs/command-reference/strings/mset.md b/docs/command-reference/strings/mset.md index e576b005..a1decbc6 100644 --- a/docs/command-reference/strings/mset.md +++ b/docs/command-reference/strings/mset.md @@ -3,6 +3,7 @@ description: Learn the proper use of Redis MSET to set multiple keys to multiple --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # MSET @@ -82,6 +83,29 @@ dragonfly$> MGET key1 key2 key3 3) "third_value" ``` + +## Benchmark + + + + ## Best Practices - Use `MSET` when you need to update multiple keys at once, as it's more efficient than separate `SET` commands. diff --git a/docs/command-reference/strings/set.md b/docs/command-reference/strings/set.md index 3feefe6c..48a2a701 100644 --- a/docs/command-reference/strings/set.md +++ b/docs/command-reference/strings/set.md @@ -3,6 +3,7 @@ description: Discover how to use Redis SET command to attach a value to a specif --- import PageTitle from '@site/src/components/PageTitle'; +import Benchmark from '@site/src/components/Benchmark'; # SET @@ -140,6 +141,29 @@ dragonfly$> TTL mykey (integer) 8 # Previous TTL remains unchanged. ``` + +## Benchmark + + + + ## Best Practices - Use `NX` or `XX` when you want to conditionally set values, such as when implementing locking or caching mechanisms. diff --git a/package.json b/package.json index d13515b8..3ef3edc7 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "write-heading-ids": "yarn run docusaurus write-heading-ids", "pull-redis-docs": "ts-node ./scripts/pull-redis-docs.ts", "generate-compatibility-json": "ts-node ./scripts/generate-compatibility-json.ts", + "sync-benchmarks": "ts-node ./scripts/sync-benchmarks.ts", "dragonfly-latest-version": "curl -s https://version.dragonflydb.io/v1 | grep -oE '[0-9.]+'" }, "dependencies": { diff --git a/scripts/sync-benchmarks.ts b/scripts/sync-benchmarks.ts new file mode 100644 index 00000000..d331d3bd --- /dev/null +++ b/scripts/sync-benchmarks.ts @@ -0,0 +1,407 @@ +#!/usr/bin/env ts-node +// +// Regenerates the `## Benchmark` section on command-reference pages from +// the latest results published in github.com/dragonflydb/benchmarking. +// +// Usage: yarn sync-benchmarks +// Intended to run on a schedule via .github/workflows/sync-benchmarks.yml, +// which opens a PR only when the generated content actually changes. + +import AdmZip from "adm-zip"; +import fetch from "cross-fetch"; +import fs from "fs"; +import path from "path"; + +const BENCHMARKING_ZIP = + "https://github.com/dragonflydb/benchmarking/archive/refs/heads/main.zip"; +const BENCHMARKING_BLOB_ROOT = + "https://github.com/dragonflydb/benchmarking/blob/main/"; +const DOCS_COMMAND_REFERENCE = path.join( + __dirname, + "../docs/command-reference" +); +const GITHUB_API = "https://api.github.com"; + +const START_MARKER = ""; +const END_MARKER = ""; + +const KEY_DIST_LABELS: Record = { + U: "uniform", + N: "normal", + Z: "zipfian", + S: "sequential", +}; + +type ResultRow = { + engine: string; + throughput: string; + p50: string; + p99: string; + p999: string; + avgLatency: string; +}; + +type BenchmarkData = { + command: string; + dragonflyOps: number; + valkeyOps: number; + redisOps: number; + hardware: string; + tool: string; + client: string; + dataset: string; + duration: string; + measuredOn: string; + harnessPath: string; + results: ResultRow[]; +}; + +function githubHeaders(): Record { + const token = process.env.GITHUB_TOKEN; + return token ? { Authorization: `Bearer ${token}` } : {}; +} + +async function fetchRepoZip(url: string): Promise { + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`); + } + const buffer = Buffer.from(await res.arrayBuffer()); + return new AdmZip(buffer); +} + +// Pulls out every `--flag value` (or bare `--flag`) token from a shell-ish +// dfbench invocation, tolerating backslash line continuations. +function parseFlags(block: string): Record { + const flags: Record = {}; + const re = /--([a-zA-Z][\w-]*)(?:[ \t]+(?:"([^"]*)"|(\S+)))?/g; + let match: RegExpExecArray | null; + while ((match = re.exec(block))) { + const [, name, quoted, bare] = match; + const value = quoted ?? bare ?? ""; + // Skip when the "value" is actually the next flag (bare boolean flag). + flags[name] = value.startsWith("--") ? "" : value; + } + return flags; +} + +function section(md: string, heading: string): string | null { + const start = md.indexOf(heading); + if (start === -1) return null; + const rest = md.slice(start + heading.length); + const next = rest.search(/\n#{2,3} /); + return next === -1 ? rest : rest.slice(0, next); +} + +function parseExpectedResults(md: string): ResultRow[] { + const body = section(md, "Expected results"); + if (!body) throw new Error("No 'Expected results' section found"); + + const rows: ResultRow[] = []; + for (const line of body.split("\n")) { + if (!line.trim().startsWith("|")) continue; + const cells = line + .split("|") + .map((c) => c.trim().replace(/^\*\*|\*\*$/g, "")) + .filter((c) => c.length > 0); + if (cells.length < 6) continue; + const [engineRaw, throughput, p50, p99, p999, avgLatency] = cells; + if (/^-+$/.test(throughput)) continue; // markdown table separator row + const engine = engineRaw.toLowerCase(); + if (!["dragonfly", "redis", "valkey"].includes(engine)) continue; + rows.push({ + engine: engine[0].toUpperCase() + engine.slice(1), + throughput, + p50, + p99, + p999, + avgLatency, + }); + } + if (rows.length === 0) throw new Error("Could not parse any result rows"); + return rows; +} + +function opsFromThroughput(throughput: string): number { + const m = /^([\d.]+)\s*([MK]?)/.exec(throughput); + if (!m) throw new Error(`Unrecognized throughput value: ${throughput}`); + const [, num, unit] = m; + const multiplier = unit === "M" ? 1_000_000 : unit === "K" ? 1_000 : 1; + return Math.round(parseFloat(num) * multiplier); +} + +function buildDataset( + tool: "dfly_bench" | "memtier", + flags: Record, + usesValuePayload: boolean +): string { + const parts: string[] = []; + const keyMax = flags["command-key-maximum"] ?? flags["key-maximum"]; + const preloadItems = Number(flags["preload-items"] ?? 0); + + if (keyMax) { + const count = formatCount(keyMax); + parts.push( + preloadItems > 1 ? `${count} keys, ${preloadItems} preloaded items each` : `${count} keys` + ); + } + + const dataSize = flags["dfly-bench-data-size"] ?? flags["memtier-data-size"]; + if (usesValuePayload && dataSize) { + parts.push(`${dataSize}B values`); + } + + if (tool === "dfly_bench") { + const distLabel = KEY_DIST_LABELS[flags["dfly-bench-key-dist"]]; + if (distLabel) parts.push(`${distLabel} key distribution`); + } + + return parts.join(", "); +} + +function formatCount(raw: string): string { + const n = Number(raw); + if (!Number.isFinite(n)) return raw; + if (n >= 1_000_000 && n % 1_000_000 === 0) return `${n / 1_000_000}M`; + if (n >= 1_000 && n % 1_000 === 0) return `${n / 1_000}K`; + return String(n); +} + +function usesValue(flags: Record): boolean { + if (flags["command"]) return true; // builtin GET/SET always read or write a value + const template = flags["command-template"] ?? ""; + const preload = flags["preload-template"] ?? ""; + return template.includes("__data__") || preload.includes("__data__"); +} + +async function fetchMeasuredOn(harnessPath: string): Promise { + const url = `${GITHUB_API}/repos/dragonflydb/benchmarking/commits?path=${encodeURIComponent( + harnessPath + )}&per_page=1`; + const res = await fetch(url, { headers: githubHeaders() }); + if (!res.ok) { + console.warn(` Could not fetch commit date for ${harnessPath} (${res.status}); using today.`); + return new Date().toISOString().slice(0, 10); + } + const commits = await res.json(); + const date = commits?.[0]?.commit?.author?.date; + return date ? date.slice(0, 10) : new Date().toISOString().slice(0, 10); +} + +async function parseCommand( + command: string, + tool: "dfly_bench" | "memtier", + harnessPath: string, + md: string +): Promise { + const setup = section(md, "Stateful setup:") ?? ""; + const testRun = section(md, "Test run:") ?? ""; + const setupFlags = parseFlags(setup); + const runFlags = parseFlags(testRun); + + const hardware = `Server: ${setupFlags["server-instance"]} (${setupFlags["server-arch"]}) · Client: ${setupFlags["client-instance"]} (${setupFlags["client-arch"]})`; + + const client = + tool === "dfly_bench" + ? `${runFlags["dfly-bench-threads"]} threads, ${runFlags["dfly-bench-conns"]} connections, pipeline ${runFlags["dfly-bench-pipeline"]}` + : `${runFlags["memtier-threads"]} threads, ${runFlags["memtier-clients"]} clients, pipeline ${runFlags["memtier-pipeline"]}`; + + const dataset = buildDataset(tool, runFlags, usesValue(runFlags)); + + const testTime = runFlags["test-time"]; + const warmupTime = runFlags["warmup-time"]; + const trials = Number(runFlags["trials"] ?? 1); + const duration = `${testTime}s (${warmupTime}s warmup), ${trials} trial${trials === 1 ? "" : "s"}`; + + const results = parseExpectedResults(md); + const opsByEngine = Object.fromEntries( + results.map((r) => [r.engine, opsFromThroughput(r.throughput)]) + ); + + const ordered = ["Dragonfly", "Valkey", "Redis"] + .map((name) => results.find((r) => r.engine === name)) + .filter((r): r is ResultRow => Boolean(r)); + + return { + command, + dragonflyOps: opsByEngine["Dragonfly"], + valkeyOps: opsByEngine["Valkey"], + redisOps: opsByEngine["Redis"], + hardware, + tool, + client, + dataset, + duration, + measuredOn: await fetchMeasuredOn(harnessPath), + harnessPath, + results: ordered, + }; +} + +function renderBenchmarkBlock(data: BenchmarkData): string { + const resultsLines = data.results + .map( + (r) => + ` { engine: "${r.engine}", throughput: "${r.throughput}", p50: "${r.p50}", p99: "${r.p99}", p999: "${r.p999}", avgLatency: "${r.avgLatency}" },` + ) + .join("\n"); + + return [ + START_MARKER, + "## Benchmark", + "", + "", + END_MARKER, + ].join("\n"); +} + +// Where to insert a Benchmark section on a page that doesn't have one yet. +const INSERT_BEFORE_HEADINGS = [ + "## Best Practices", + "## Common Mistakes", + "## FAQs", + "## See also", + "## Pattern:", +]; + +function upsertBenchmarkBlock(fileContents: string, block: string): string { + const markerRe = new RegExp( + `${START_MARKER}[\\s\\S]*?${END_MARKER}`, + "m" + ); + if (markerRe.test(fileContents)) { + return fileContents.replace(markerRe, block); + } + + // Legacy (pre-marker) block: `## Benchmark` heading directly followed by + // a single self-closing tag, with no markers. + const legacyRe = /## Benchmark\n\n\n?/m; + if (legacyRe.test(fileContents)) { + return fileContents.replace(legacyRe, `${block}\n`); + } + + for (const heading of INSERT_BEFORE_HEADINGS) { + const idx = fileContents.indexOf(`\n${heading}`); + if (idx !== -1) { + return `${fileContents.slice(0, idx)}\n${block}\n${fileContents.slice(idx + 1)}`; + } + } + + return `${fileContents.trimEnd()}\n\n${block}\n`; +} + +function ensureBenchmarkImport(fileContents: string): string { + if (fileContents.includes("@site/src/components/Benchmark")) { + return fileContents; + } + const pageTitleImport = "import PageTitle from '@site/src/components/PageTitle';"; + if (fileContents.includes(pageTitleImport)) { + return fileContents.replace( + pageTitleImport, + `${pageTitleImport}\nimport Benchmark from '@site/src/components/Benchmark';` + ); + } + // No PageTitle import (unusual, but be defensive): add right after front matter. + return fileContents.replace( + /^---\n[\s\S]*?\n---\n/, + (fm) => `${fm}\nimport Benchmark from '@site/src/components/Benchmark';\n` + ); +} + +function findDocFile(command: string): string | null { + const target = `${command.toLowerCase()}.md`; + const stack = [DOCS_COMMAND_REFERENCE]; + while (stack.length) { + const dir = stack.pop()!; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (entry.name.toLowerCase() === target) return full; + } + } + return null; +} + +async function main() { + console.log("Fetching dragonflydb/benchmarking..."); + const zip = await fetchRepoZip(BENCHMARKING_ZIP); + + const reproduceEntries = zip + .getEntries() + .filter((e) => /\/benchmarks\/[^/]+\/(dfly_bench|memtier)\/[^/]+_reproduce\.md$/.test(e.entryName)); + + // command -> { dfly_bench?: entry, memtier?: entry } + const byCommand = new Map>>(); + for (const entry of reproduceEntries) { + const m = /\/benchmarks\/([^/]+)\/(dfly_bench|memtier)\//.exec(entry.entryName)!; + const [, command, tool] = m; + const existing = byCommand.get(command) ?? {}; + existing[tool as "dfly_bench" | "memtier"] = entry; + byCommand.set(command, existing); + } + + console.log(`Found ${byCommand.size} command(s) in benchmarking repo.`); + + let changed = 0; + let skippedNoDoc = 0; + let failed = 0; + + for (const [command, tools] of byCommand) { + // Prefer dfly_bench for consistent methodology across all commands. + const tool = tools.dfly_bench ? "dfly_bench" : "memtier"; + const entry = tools[tool]!; + + const docFile = findDocFile(command); + if (!docFile) { + console.log(`- ${command}: no matching docs/command-reference page, skipping.`); + skippedNoDoc++; + continue; + } + + const zipInternalPath = entry.entryName.replace(/^[^/]+\//, ""); // strip "-/" prefix + const harnessPath = zipInternalPath; + + try { + const md = entry.getData().toString(); + const data = await parseCommand(command, tool, harnessPath, md); + const block = renderBenchmarkBlock(data); + + const before = fs.readFileSync(docFile, "utf8"); + const after = ensureBenchmarkImport(upsertBenchmarkBlock(before, block)); + + if (after !== before) { + fs.writeFileSync(docFile, after); + console.log(`- ${command}: updated ${path.relative(process.cwd(), docFile)}`); + changed++; + } else { + console.log(`- ${command}: up to date.`); + } + } catch (err) { + console.error(`- ${command}: FAILED to sync (${(err as Error).message})`); + failed++; + } + } + + console.log( + `\nDone. ${changed} file(s) changed, ${skippedNoDoc} command(s) skipped (no doc page), ${failed} failure(s).` + ); + + if (failed > 0) process.exitCode = 1; +} + +main(); diff --git a/src/components/Benchmark/index.tsx b/src/components/Benchmark/index.tsx new file mode 100644 index 00000000..fcbf66d9 --- /dev/null +++ b/src/components/Benchmark/index.tsx @@ -0,0 +1,194 @@ +import React from "react"; +import clsx from "clsx"; + +import styles from "./styles.module.css"; + +interface BenchmarkProps { + /** Command name shown in the intro sentence, e.g. "GET". */ + command: string; + dragonflyOps: number; + redisOps: number; + valkeyOps: number; + /** Server + client instance types, e.g. "Server: m7g.8xlarge (arm64) · Client: c6gn.8xlarge (arm64)". */ + hardware: string; + /** Load-generation tool and concurrency settings. */ + client: string; + /** Keyspace / value size used for the run. */ + dataset: string; + /** Test duration, warmup, and trial count. */ + duration: string; + /** ISO date the numbers were captured. */ + measuredOn: string; + /** Path to the reproduce doc within github.com/dragonflydb/benchmarking. */ + harnessPath: string; + /** "memtier_benchmark" or "dfly_bench" — used in the closing methodology sentence. */ + tool: string; + /** Verbatim rows from the harness's "Expected results" table. */ + results: { + engine: string; + throughput: string; + p50: string; + p99: string; + p999: string; + avgLatency: string; + }[]; +} + +function formatOps(n: number): string { + return n.toLocaleString("en-US"); +} + +function formatTick(v: number): string { + if (v === 0) return "0"; + return `${(v / 1_000_000).toFixed(2).replace(/0$/, "")}M`; +} + +export default function Benchmark({ + command, + dragonflyOps, + redisOps, + valkeyOps, + hardware, + client, + dataset, + duration, + measuredOn, + harnessPath, + tool, + results, +}: BenchmarkProps): JSX.Element { + const engines = [ + { name: "Dragonfly", ops: dragonflyOps, isWinner: true }, + { name: "Valkey", ops: valkeyOps, isWinner: false }, + { name: "Redis", ops: redisOps, isWinner: false }, + ]; + + const max = Math.max(dragonflyOps, redisOps, valkeyOps); + const scale = Math.ceil(max / 500_000) * 500_000; + const ticks = [0, 1, 2, 3, 4].map((i) => formatTick((scale * i) / 4)); + + const methodology: [string, string][] = [ + ["Hardware", hardware], + ["Client", client], + ["Dataset", dataset], + ["Duration", duration], + ["Measured", measuredOn], + ]; + + const harnessUrl = `https://github.com/dragonflydb/benchmarking/blob/main/${harnessPath}`; + + return ( +
+

+ Sustained throughput for {command} on a single instance, + measured against Redis and Valkey on identical hardware. Higher is + better. +

+ +
+ {engines.map((engine) => { + const width = ((engine.ops / scale) * 100).toFixed(1) + "%"; + const relative = engine.isWinner + ? "OPS" + : (dragonflyOps / engine.ops).toFixed(1) + "× OPS"; + + return ( +
+ + {engine.name} + + + + + + + {formatOps(engine.ops)} + + {relative} + +
+ ); + })} +
+ +
+ +
+
+ {ticks.map((_, i) => ( + + ))} +
+
+ {ticks.map((label, i) => ( + {label} + ))} +
+
operations / second
+
+ +
+ + + + + + + + + + + + + + {results.map((row) => ( + + + + + + + + + ))} + +
EngineThroughputp50p99p99.9Avg Latency
+ {row.engine} + {row.throughput}{row.p50}{row.p99}{row.p999}{row.avgLatency}
+ +

Methodology

+ + + {methodology.map(([label, value]) => ( + + + + + ))} + +
{label} + {value} +
+

+ Ran through {tool} until throughput stabilized; the + reported figure is the median throughput sampled during the run. + Harness and raw output: dragonflydb/benchmarking. +

+
+ ); +} diff --git a/src/components/Benchmark/styles.module.css b/src/components/Benchmark/styles.module.css new file mode 100644 index 00000000..8ecd6030 --- /dev/null +++ b/src/components/Benchmark/styles.module.css @@ -0,0 +1,145 @@ +.benchmark { + margin: 1.5rem 0 2rem; +} + +.rows { + display: flex; + flex-direction: column; + gap: 20px; + margin-top: 1rem; +} + +.row { + display: grid; + grid-template-columns: 90px minmax(0, 1fr) 170px; + align-items: center; + gap: 16px; +} + +.name { + font-size: 0.95rem; + font-weight: 600; + color: var(--ifm-color-emphasis-700); +} + +.winnerName { + color: var(--ifm-color-primary); +} + +.track { + height: 22px; + border-radius: 5px; + background: var(--ifm-color-emphasis-200); + overflow: hidden; +} + +.bar { + display: block; + height: 100%; + border-radius: 5px; + background: var(--ifm-color-emphasis-400); +} + +.winnerBar { + background: var(--brand-gradient); +} + +.value { + display: flex; + align-items: baseline; + justify-content: flex-end; + gap: 8px; + font-family: var(--ifm-font-family-monospace); + font-size: 0.95rem; + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +.relative { + font-size: 0.8rem; + color: var(--ifm-color-emphasis-500); +} + +.axis { + display: grid; + grid-template-columns: 90px minmax(0, 1fr) 170px; + gap: 16px; + margin: 12px 0 0; +} + +.axisTicks { + display: flex; + justify-content: space-between; + height: 5px; + border-top: 1px solid var(--ifm-color-emphasis-300); +} + +.axisTicks span { + width: 1px; + height: 4px; + background: var(--ifm-color-emphasis-300); +} + +.axisLabels { + display: flex; + justify-content: space-between; + font-family: var(--ifm-font-family-monospace); + font-size: 0.7rem; + color: var(--ifm-color-emphasis-500); +} + +.axisCaption { + font-size: 0.72rem; + color: var(--ifm-color-emphasis-500); + text-align: center; + margin-top: 4px; +} + +.resultsTable { + width: 100%; + margin: 1.25rem 0 2rem; + font-size: 0.85rem; + font-family: var(--ifm-font-family-monospace); + font-variant-numeric: tabular-nums; +} + +.resultsTable th, +.resultsTable td { + border: 1px solid var(--ifm-table-border-color); + padding: 0.4rem 0.75rem; + text-align: right; +} + +.resultsTable th:first-child, +.resultsTable td:first-child { + text-align: left; + font-family: var(--ifm-font-family-base); +} + +.methodologyTable { + width: 100%; + margin: 0.75rem 0 0; + font-size: 0.9rem; +} + +.methodologyTable tr { + background: none; +} + +.methodologyTable td { + border: 1px solid var(--ifm-table-border-color); + padding: 0.5rem 0.75rem; + vertical-align: top; +} + +.methodologyLabel { + width: 168px; + color: var(--ifm-color-emphasis-600); + white-space: nowrap; +} + +.footnote { + font-size: 0.9rem; + color: var(--ifm-color-emphasis-700); + margin: 0.75rem 0 0; +}