diff --git a/.github/workflows/publish-cfdocs-snippets.yml b/.github/workflows/publish-cfdocs-snippets.yml new file mode 100644 index 0000000000..83513f207e --- /dev/null +++ b/.github/workflows/publish-cfdocs-snippets.yml @@ -0,0 +1,63 @@ +name: Publish CF Docs Snippets + +on: + workflow_dispatch: + + push: + branches: + - main + paths: + - docs/src/** + - apps/app/src/pack/examples/** + - cluster/helm/** + - gha-scripts/cf-docs/** + +permissions: + actions: write + contents: read + +jobs: + publish_cfdocs_snippets: + # Requires repository variable ENABLE_SYNC_PROCESS=true (see source-repo-workflow-readme.md). + if: vars.ENABLE_SYNC_PROCESS == 'true' + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Extract snippet data + run: node gha-scripts/cf-docs/generateOutputDocs.js + + - name: Store artifact output + id: store-artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ github.event.repository.name }}-snippets + path: docs-output/ + + - name: Prepare additional params + id: additional-params + run: echo "short_sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + + - name: Generate snippet writer app token + id: writer-token + uses: actions/create-github-app-token@3a38478537b0cedb864eaff129aacbe19810c233 # v2.2.1 + with: + app-id: ${{ secrets.CF_DOCS_SNIPPET_WRITER_APP_ID }} + private-key: ${{ secrets.CF_DOCS_SNIPPET_WRITER_PRIVATE_KEY }} + owner: ${{ vars.MAIN_REPO_ORG }} + repositories: ${{ vars.MAIN_REPO_NAME }} + + - name: Update main docs repo + uses: passeidireto/trigger-external-workflow-action@main + env: + PAYLOAD_run-id: ${{ github.run_id }} + PAYLOAD_artifact-id: ${{ steps.store-artifact.outputs.artifact-id }} + PAYLOAD_repo-name: ${{ github.event.repository.name }} + PAYLOAD_repo-org: ${{ github.repository_owner }} + PAYLOAD_repo-version: main + PAYLOAD_trigger_sha_short: ${{ steps.additional-params.outputs.short_sha }} + with: + repository: ${{ vars.MAIN_REPO_ORG }}/${{ vars.MAIN_REPO_NAME }} + event: update_snippets + github_pat: ${{ steps.writer-token.outputs.token }} diff --git a/gha-scripts/cf-docs/exportConfig.json b/gha-scripts/cf-docs/exportConfig.json new file mode 100644 index 0000000000..f4ed56bd1c --- /dev/null +++ b/gha-scripts/cf-docs/exportConfig.json @@ -0,0 +1,193 @@ +{ + "rstIncludeRefTargets": { + "validator-backups": "/global-synchronizer/production-operations/validator-backups", + "sv_backups": "/global-synchronizer/production-operations/sv-backup", + "traffic": "/global-synchronizer/deployment/synchronizer-traffic", + "traffic_topup": "/global-synchronizer/deployment/synchronizer-traffic#traffic-top-ups-how-does-one-buy-traffic", + "traffic_parameters": "/global-synchronizer/deployment/synchronizer-traffic#traffic-parameters", + "traffic parameters": "/global-synchronizer/deployment/synchronizer-traffic#traffic-parameters", + "traffic_accounting": "/global-synchronizer/deployment/synchronizer-traffic#traffic-accounting-what-counts-as-traffic" + }, + "urlSubstitutions": { + "https://docs.daml.com/canton/usermanual/kms/kms_aws_setup.html": "[Canton KMS operations](/global-synchronizer/production-operations/kms-operations#configure-a-amazon-web-services-aws-kms)", + "https://docs.daml.com/canton/usermanual/kms/kms_gcp_setup.html": "[Canton KMS operations](/global-synchronizer/production-operations/kms-operations#configure-a-google-cloud-provider-gcp-kms)" + }, + "snippets": [ + { + "snippetName": "splice-literal-full-apps-app-src-pack-examples-recovery-manual-identities-dump", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/recovery/manual-identities-dump.sc", + "location": { + "type": "fullFile" + }, + "description": "", + "options": { + "language": "scala" + } + }, + { + "snippetName": "splice-literal-full-apps-app-src-pack-examples-sv-helm-kms-participant-aws-values", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/kms-participant-aws-values.yaml", + "location": { + "type": "fullFile" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-full-apps-app-src-pack-examples-sv-helm-kms-participant-gcp-values", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/kms-participant-gcp-values.yaml", + "location": { + "type": "fullFile" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-standalone-validator-values-configuring-topup-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/standalone-validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "CONFIGURING_TOPUP_START", + "end": "CONFIGURING_TOPUP_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-standalone-validator-values-participant-bootstrap-migrate-to-new-participant-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/standalone-validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "PARTICIPANT_BOOTSTRAP_MIGRATE_TO_NEW_PARTICIPANT_START", + "end": "PARTICIPANT_BOOTSTRAP_MIGRATE_TO_NEW_PARTICIPANT_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-standalone-validator-values-scan-client-configuration-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/standalone-validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "SCAN_CLIENT_CONFIGURATION_START", + "end": "SCAN_CLIENT_CONFIGURATION_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-standalone-validator-values-synchronizer-configuration-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/standalone-validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "SYNCHRONIZER_CONFIGURATION_START", + "end": "SYNCHRONIZER_CONFIGURATION_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-sv-validator-values-sv-participant-pruning-schedule-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/sv-validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "SV_PARTICIPANT_PRUNING_SCHEDULE_START", + "end": "SV_PARTICIPANT_PRUNING_SCHEDULE_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-sv-values-docs-pruning-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/sv-values.yaml", + "location": { + "type": "stringMarker", + "start": "DOCS_PRUNING_START", + "end": "DOCS_PRUNING_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-validator-values-auto-accept-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "AUTO_ACCEPT_START", + "end": "AUTO_ACCEPT_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-validator-values-enablewallet-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "ENABLEWALLET_START", + "end": "ENABLEWALLET_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-apps-app-src-pack-examples-sv-helm-validator-values-participant-pruning-schedule-start", + "sourceRepo": "splice", + "sourceFilepath": "apps/app/src/pack/examples/sv-helm/validator-values.yaml", + "location": { + "type": "stringMarker", + "start": "PARTICIPANT_PRUNING_SCHEDULE_START", + "end": "PARTICIPANT_PRUNING_SCHEDULE_END" + }, + "description": "", + "options": { + "language": "yaml" + } + }, + { + "snippetName": "splice-literal-marker-cluster-helm-splice-cometbft-values-template-docs-cometbft-pruning-start", + "sourceRepo": "splice", + "sourceFilepath": "cluster/helm/splice-cometbft/values-template.yaml", + "location": { + "type": "stringMarker", + "start": "DOCS_COMETBFT_PRUNING_START", + "end": "DOCS_COMETBFT_PRUNING_END" + }, + "description": "", + "options": { + "language": "yaml" + } + } + ] +} diff --git a/gha-scripts/cf-docs/generateOutputDocs.js b/gha-scripts/cf-docs/generateOutputDocs.js new file mode 100644 index 0000000000..52439dcf75 --- /dev/null +++ b/gha-scripts/cf-docs/generateOutputDocs.js @@ -0,0 +1,892 @@ +#!/usr/bin/env node + +// generateOutputDocs.js +// +// - Reads a single export config (default path varies by repo; override with --export-config) +// - Writes extracted snippets into docs-output/.mdx (override with --output) +// - Resolves source files relative to the repo root (override with --repo-root) +// +// Optional CLI overrides: +// --repo-root +// --export-config +// --output +// --verbose + +const fs = require('fs') +const path = require('path') + +const DEFAULT_REPO_ROOT = path.join(__dirname, '..', '..') +const DEFAULT_EXPORT_CONFIG_PATH = path.join(__dirname, 'exportConfig.json') +const DEFAULT_OUTPUT_FOLDER_PATH = path.join(DEFAULT_REPO_ROOT, 'docs-output') + +function printUsage() { + console.log(`Usage: node generateOutputDocs.js [options] + +Options: + --repo-root Repository root for resolving snippet source files + (default: ${DEFAULT_REPO_ROOT}) + --export-config Export config JSON file + (default: ${DEFAULT_EXPORT_CONFIG_PATH}) + --output Output directory for generated .mdx snippets + (default: ${DEFAULT_OUTPUT_FOLDER_PATH}) + --verbose Log each snippet as it is processed + -h, --help Show this help message +`) +} + +function parseCli(argv) { + let repoRoot = DEFAULT_REPO_ROOT + let exportConfigPath = DEFAULT_EXPORT_CONFIG_PATH + let outputFolderPath = DEFAULT_OUTPUT_FOLDER_PATH + let verbose = false + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + if (arg === '--verbose') { + verbose = true + continue + } + if (arg === '--help' || arg === '-h') { + printUsage() + process.exit(0) + } + if (arg === '--repo-root') { + const value = argv[++i] + if (!value) { + throw new Error('--repo-root requires a path argument') + } + repoRoot = path.resolve(value) + continue + } + if (arg === '--export-config') { + const value = argv[++i] + if (!value) { + throw new Error('--export-config requires a path argument') + } + exportConfigPath = path.resolve(value) + continue + } + if (arg === '--output') { + const value = argv[++i] + if (!value) { + throw new Error('--output requires a path argument') + } + outputFolderPath = path.resolve(value) + continue + } + throw new Error(`Unknown argument: ${arg}`) + } + + return { + repoRoot, + exportConfigPath, + outputFolderPath, + verbose, + } +} + +function readFileContent(filePath) { + try { + return fs.readFileSync(filePath, 'utf8') + } catch (error) { + throw new Error(`Failed to read file ${filePath}: ${error.message}`) + } +} + +function extractByLines(fileContent, start, end) { + const lines = fileContent.split(/\r?\n/) + const startLine = Number(start) + const endLine = Number(end) + + if ( + startLine < 1 || + endLine < 1 || + startLine > lines.length || + endLine > lines.length + ) { + throw new Error( + `Line numbers out of range: start=${startLine}, end=${endLine}, file has ${lines.length} lines` + ) + } + + if (startLine > endLine) { + throw new Error( + `Invalid line range: start (${startLine}) must be <= end (${endLine})` + ) + } + + return lines.slice(startLine - 1, endLine).join('\n') +} + +function extractByStringMarker(fileContent, startMarker, endMarker) { + const startIndex = fileContent.indexOf(startMarker) + if (startIndex === -1) { + throw new Error(`Start marker not found: "${startMarker}"`) + } + + // Match Sphinx literalinclude :start-after: / :end-before: — exclude marker lines. + let contentStart = fileContent.indexOf('\n', startIndex) + if (contentStart === -1) { + contentStart = startIndex + startMarker.length + } else { + contentStart += 1 + } + + const endIndex = fileContent.indexOf(endMarker, contentStart) + if (endIndex === -1) { + throw new Error(`End marker not found: "${endMarker}"`) + } + + let contentEnd = fileContent.lastIndexOf('\n', endIndex) + if (contentEnd < contentStart) { + contentEnd = endIndex + } + + return fileContent.substring(contentStart, contentEnd).trim() +} + +function extractByRegexWrap(fileContent, startRegex, endRegex) { + const startPattern = new RegExp(startRegex) + const endPattern = new RegExp(endRegex) + + const startMatch = fileContent.match(startPattern) + if (!startMatch) { + throw new Error(`Start regex pattern not found: "${startRegex}"`) + } + + const contentStart = startMatch.index + startMatch[0].length + const remainingContent = fileContent.substring(contentStart) + const endMatch = remainingContent.match(endPattern) + + if (!endMatch) { + throw new Error(`End regex pattern not found: "${endRegex}"`) + } + + return remainingContent.substring(0, endMatch.index).trim() +} + +function extractByJsonIndex(fileContent, start, end) { + let arr + try { + arr = JSON.parse(fileContent) + } catch (e) { + throw new Error(`File is not valid JSON: ${e.message}`) + } + if (!Array.isArray(arr)) { + throw new Error( + 'JSON root must be an array for location type jsonIndex' + ) + } + const startIdx = Number(start) + const endIdx = Number(end) + if ( + startIdx < 0 || + endIdx < 0 || + startIdx >= arr.length || + endIdx >= arr.length + ) { + throw new Error( + `Array index out of range: start=${startIdx}, end=${endIdx}, array length=${arr.length}` + ) + } + if (startIdx > endIdx) { + throw new Error( + `Invalid index range: start (${startIdx}) must be <= end (${endIdx})` + ) + } + if (startIdx === endIdx) { + const item = arr[startIdx] + return typeof item === 'string' ? item : String(item) + } + return arr + .slice(startIdx, endIdx + 1) + .map((item) => (typeof item === 'string' ? item : String(item))) + .join('\n') +} + +function extractSnippetContent(fileContent, location) { + switch (location.type) { + case 'fullFile': + return fileContent + + case 'lines': + return extractByLines(fileContent, location.start, location.end) + + case 'jsonIndex': + return extractByJsonIndex(fileContent, location.start, location.end) + + case 'stringMarker': + return extractByStringMarker( + fileContent, + location.start, + location.end + ) + + case 'regexWrap': + return extractByRegexWrap(fileContent, location.start, location.end) + + default: + throw new Error(`Unknown location type: ${location.type}`) + } +} + +function normalizeIndent(content) { + const lines = content.split('\n') + + let minIndent = null + for (const line of lines) { + if (line.trim() === '') continue + const match = line.match(/^(\s*)/) + const indent = match ? match[1].length : 0 + if (minIndent === null || indent < minIndent) { + minIndent = indent + } + } + + // Strip the common leading whitespace from every non-blank line and then + // re-indent the whole block by two spaces. Using `line.slice(strip)` + // (instead of stripping ALL leading whitespace) preserves the relative + // indentation between lines — including the case where minIndent is 0, + // which would otherwise flatten any source that contains a top-level + // line at column 0 (e.g. HOCON config files where a `}` closes at the + // start of the line). + const strip = minIndent ?? 0 + return lines + .map((line) => { + if (line.trim() === '') return '' + return ` ${line.slice(strip)}` + }) + .join('\n') +} + +/** Strip common leading indent only; first line starts at column 0 (HOCON/config in RST). */ +function baselineIndent(content) { + const lines = content.split('\n') + let minIndent = null + for (const line of lines) { + if (line.trim() === '') continue + const indent = (line.match(/^(\s*)/) || ['', ''])[1].length + if (minIndent === null || indent < minIndent) minIndent = indent + } + const strip = minIndent ?? 0 + return lines + .map((line) => (line.trim() === '' ? '' : line.slice(strip))) + .join('\n') +} + +function applyIndentOption(content, normalizeIndentOption) { + if (normalizeIndentOption === false) return content + if (normalizeIndentOption === 'baseline') return baselineIndent(content) + return normalizeIndent(content) +} + +/** + * Resolve indent mode for a snippet. Crawl configs historically set bash to false + * (preserve RST indent); we want column-0 commands via baseline instead. + */ +function resolveNormalizeIndent(snippet) { + const opt = snippet.options?.normalizeIndent + const lang = (snippet.options?.language || '').toLowerCase() + if (opt === false && (lang === 'bash' || lang === 'parsed-literal')) { + return 'baseline' + } + if (opt !== undefined) return opt + return defaultNormalizeIndent(snippet.location) +} + +/** Default indent mode when options.normalizeIndent is omitted. */ +function defaultNormalizeIndent(location) { + switch (location && location.type) { + case 'fullFile': + case 'stringMarker': + return 'baseline' + default: + return true + } +} + +function trimBlankEdges(content) { + return content.replace(/^\s*\n+/, '').replace(/\n+\s*$/, '') +} + +// --- RST include → MDX (options.transform === "rstinclude") --- + +function stripLeadingRstComments(lines) { + let i = 0 + while (i < lines.length) { + const t = lines[i].trim() + if (t === '') { + i++ + continue + } + if (/^\.\./.test(t) && !/^\.\.\s+[\w-]+::/.test(t)) { + i++ + continue + } + if (/^Copyright\b/i.test(t) || /^SPDX-License-Identifier:/i.test(t)) { + i++ + continue + } + break + } + return lines.slice(i) +} + +function dedentBlock(lines) { + if (lines.length === 0) return [] + const indents = lines + .filter((l) => l.trim() !== '') + .map((l) => (l.match(/^(\s*)/) || ['', ''])[1].length) + const min = indents.length ? Math.min(...indents) : 0 + return lines.map((l) => (l.trim() === '' ? '' : l.slice(min))) +} + +function inlineRstMarkup(text, refTargets = {}) { + let out = text + out = out.replace( + /:ref:`([^<]+?)\s*<([^>]+)>`/g, + (_, label, target) => { + const href = refTargets[target.trim()] + return href ? `[${label.trim()}](${href})` : label.trim() + } + ) + out = out.replace(/:ref:`([^`]+)`/g, (_, target) => { + const key = target.trim() + const href = refTargets[key] + if (href) { + const label = key.replace(/_/g, ' ') + return `[${label}](${href})` + } + return key + }) + out = out.replace(/\*\*([^*]+)\*\*/g, '**$1**') + out = out.replace(/``([^`]+)``/g, '`$1`') + return out +} + +function fieldListToMarkdown(lines, startIdx, refTargets = {}) { + const items = [] + let i = startIdx + while (i < lines.length) { + const line = lines[i] + const lt = line.trim() + if (lt === '') { + i++ + continue + } + if (/^\.\.\s+[\w-]+::/.test(lt)) break + if (/^\s/.test(line)) break + if (!/^[A-Z][A-Z0-9_]+$/.test(lt)) break + const term = lt + i++ + const body = [] + while (i < lines.length && /^\s+/.test(lines[i])) { + const bt = lines[i].trim() + if (bt === '') { + i++ + continue + } + if (/^\.\.\s+/.test(bt)) break + body.push(bt) + i++ + } + const bodyText = inlineRstMarkup(body.join(' '), refTargets) + const linkified = bodyText + .replace( + /form \|generic_sv_url\|/g, + 'form |generic_sv_url|' + ) + .replace( + /use \|gsf_sv_url\|/g, + 'use |gsf_sv_url|' + ) + if (term === 'ONBOARDING_SECRET') { + items.push({ + type: 'paragraph', + text: `**${term}**\n${linkified}`, + }) + } else { + items.push({ + type: 'bullet', + text: `**${term}** — ${linkified}`, + }) + } + } + return { next: i, items } +} + +function readDirectiveBlock(lines, startIdx) { + const body = [] + let i = startIdx + while (i < lines.length) { + const line = lines[i] + if (line.trim() === '') { + body.push('') + i++ + continue + } + if (!/^\s/.test(line) && /^\.\.\s+/.test(line)) break + if (!/^\s/.test(line) && line.trim() !== '' && body.length > 0) break + body.push(line) + i++ + } + return { body: dedentBlock(body), next: i } +} + +function convertRstIncludeToMdx(content, options = {}) { + const refTargets = options.refTargets || {} + const rawLines = trimBlankEdges(content).split('\n') + const lines = stripLeadingRstComments(rawLines) + const out = [] + let i = 0 + + while (i < lines.length) { + const line = lines[i] + const trimmed = line.trim() + + if (trimmed === '') { + i++ + continue + } + + const warning = trimmed.match(/^\.\.\s+warning::\s*$/) + if (warning) { + const block = readDirectiveBlock(lines, i + 1) + const inner = block.body + .filter((l) => l.trim() !== '') + .map((l) => inlineRstMarkup(l.trim(), refTargets)) + .join('\n\n') + out.push(`\n\n${inner}\n\n`) + i = block.next + continue + } + + const note = trimmed.match(/^\.\.\s+note::\s*$/) + if (note) { + const block = readDirectiveBlock(lines, i + 1) + const inner = block.body + .filter((l) => l.trim() !== '') + .map((l) => inlineRstMarkup(l.trim(), refTargets)) + .join('\n\n') + out.push(`\n\n${inner}\n\n`) + i = block.next + continue + } + + const tip = trimmed.match(/^\.\.\s+tip::\s*$/) + if (tip) { + const block = readDirectiveBlock(lines, i + 1) + const inner = block.body + .filter((l) => l.trim() !== '') + .map((l) => inlineRstMarkup(l.trim(), refTargets)) + .join('\n\n') + // Mintlify doesn't have a Tip component; render as Note. + out.push(`\n\n${inner}\n\n`) + i = block.next + continue + } + + const admonition = trimmed.match(/^\.\.\s+admonition::\s*(.+)\s*$/) + if (admonition) { + const title = admonition[1].trim() + const block = readDirectiveBlock(lines, i + 1) + const parts = [] + let j = 0 + const bodyLines = block.body + while (j < bodyLines.length) { + const bl = bodyLines[j].trim() + if (bl === '') { + j++ + continue + } + if (/^\.\.\s+parsed-literal::\s*$/.test(bl)) { + j++ + const codeBlock = readDirectiveBlock(bodyLines, j) + const code = codeBlock.body + .filter((l) => l.trim() !== '') + .join('\n') + parts.push('```bash\n' + code + '\n```') + j = codeBlock.next + continue + } + const para = [] + while (j < bodyLines.length && bodyLines[j].trim() !== '') { + if (/^\.\.\s+/.test(bodyLines[j].trim())) break + para.push(bodyLines[j].trim()) + j++ + } + if (para.length) { + parts.push(inlineRstMarkup(para.join(' '), refTargets)) + } + } + const wrapper = + title.toLowerCase().includes('devnet') || + title.toLowerCase().includes('devnet-only') + ? '
\n\n' + : '' + const wrapperEnd = wrapper ? '\n\n
' : '' + const inner = parts + .map((p) => + p.startsWith('```') ? p : `\n\n${p}\n\n` + ) + .join('\n\n') + out.push(`${wrapper}${inner}${wrapperEnd}`) + i = block.next + continue + } + + const parsedLiteral = trimmed.match(/^\.\.\s+parsed-literal::\s*$/) + if (parsedLiteral) { + const block = readDirectiveBlock(lines, i + 1) + const code = block.body + .filter((l) => l.trim() !== '') + .join('\n') + out.push('```bash\n' + code + '\n```') + i = block.next + continue + } + + const codeBlock = trimmed.match(/^\.\.\s+code(?:-block)?::\s*(\S*)\s*$/) + if (codeBlock) { + const lang = codeBlock[1] || 'bash' + const block = readDirectiveBlock(lines, i + 1) + const code = block.body + .filter((l) => l.trim() !== '') + .join('\n') + out.push('```' + lang + '\n' + code + '\n```') + i = block.next + continue + } + + if (/^\.\.\s+/.test(trimmed)) { + i++ + continue + } + + if (!/^\s/.test(line) && /^[A-Z][A-Z0-9_]+$/.test(trimmed)) { + const fl = fieldListToMarkdown(lines, i, refTargets) + let inList = false + for (const item of fl.items) { + if (item.type === 'bullet') { + if (!inList) { + out.push('
    ') + inList = true + } + out.push(`
  • ${item.text}
    • `) + } else { + if (inList) { + out.push('
') + inList = false + } + out.push(`

${item.text.replace(/\n/g, '
\n')}

`) + } + } + if (inList) out.push('') + i = fl.next + continue + } + + if (trimmed.startsWith('- ')) { + const items = [] + while (i < lines.length) { + const t = lines[i].trim() + if (t.startsWith('- ')) { + items.push( + inlineRstMarkup(t.replace(/^-\s+/, ''), refTargets) + ) + i++ + } else if (t === '') { + i++ + if ( + i >= lines.length || + (!lines[i].trim().startsWith('- ') && + !/^\s+/.test(lines[i])) + ) { + break + } + } else if (/^\s/.test(lines[i]) && items.length > 0) { + items[items.length - 1] += + ' ' + inlineRstMarkup(t, refTargets) + i++ + } else { + break + } + } + out.push('
    ') + for (const item of items) { + out.push(`
  • ${item}
    • `) + } + out.push('
') + continue + } + + if (!/^\s/.test(line)) { + const para = [] + while (i < lines.length) { + const pl = lines[i] + const pt = pl.trim() + if (pt === '') break + if (/^\.\.\s+/.test(pt)) break + if (pt.startsWith('- ')) break + if (!/^\s/.test(pl) && /^[A-Z][A-Z0-9_]+$/.test(pt)) { + break + } + para.push(pt) + i++ + } + if (para.length) { + out.push( + `

${inlineRstMarkup(para.join(' '), refTargets)}

` + ) + } + continue + } + + i++ + } + + return trimBlankEdges(out.join('\n\n')) +} + +/** Replace literal substrings (e.g. legacy docs.daml.com URLs in YAML comments). */ +function applyUrlSubstitutions(content, globalSubstitutions, snippetSubstitutions) { + const merged = { + ...(globalSubstitutions || {}), + ...(snippetSubstitutions || {}), + } + const keys = Object.keys(merged) + if (keys.length === 0) return content + let result = content + for (const from of keys) { + result = result.split(from).join(merged[from]) + } + return result +} + +function convertRstBlocksToMarkdown(content, fallbackLanguage = '') { + const input = trimBlankEdges(content) + const lines = input.split('\n') + const out = [] + let i = 0 + + while (i < lines.length) { + const m = lines[i].match(/^\s*\.\.\s+code-block::\s*(\S*)\s*$/) + if (!m) { + i++ + continue + } + + let language = (m[1] || '').trim() + if (!language || language.toLowerCase() === 'none') { + language = fallbackLanguage || '' + } + + i++ + while (i < lines.length && lines[i].trim() === '') i++ + + const block = [] + while (i < lines.length) { + const line = lines[i] + if (line.trim() === '') { + block.push('') + i++ + continue + } + + if (/^( {4}|\t)/.test(line)) { + block.push(line.replace(/^( {4}|\t)/, '')) + i++ + continue + } + break + } + + while (block.length > 0 && block[block.length - 1] === '') { + block.pop() + } + + if (language) { + out.push(`\`\`\`${language}`) + } else { + out.push('```') + } + out.push(block.join('\n')) + out.push('```') + out.push('') + } + + if (out.length === 0) { + // Safety fallback: strip any leftover RST directives and keep only content. + const cleaned = input + .split('\n') + .filter((line) => !/^\s*\.\.\s+code-block::/.test(line)) + .join('\n') + const trimmed = trimBlankEdges(cleaned) + const language = fallbackLanguage || '' + if (language) { + return `\`\`\`${language}\n${trimmed}\n\`\`\`` + } + return `\`\`\`\n${trimmed}\n\`\`\`` + } + + while (out.length > 0 && out[out.length - 1] === '') out.pop() + return out.join('\n') +} + +function formatSnippetContent(content, options, globalOptions = {}) { + let body = content + if (options && options.unescapeRstQuotes) { + body = body.replace(/\\'/g, "'") + } + if (options && options.transform === 'rstinclude') { + return convertRstIncludeToMdx(body, { + refTargets: { + ...(globalOptions.rstIncludeRefTargets || {}), + ...(options.refTargets || {}), + }, + }) + } + if (options && options.transform === 'rstjson') { + const language = options && options.language ? options.language : '' + return convertRstBlocksToMarkdown(body, language) + } + const displayStyle = (options && options.displayStyle) || 'wrapCode' + const rawLanguage = options && options.language ? options.language : '' + const language = + rawLanguage && rawLanguage.toLowerCase() === 'none' ? '' : rawLanguage + + switch (displayStyle) { + case 'wrapCode': + body = trimBlankEdges(body) + if (language) { + return `\`\`\`${language}\n${body}\n\`\`\`` + } else { + return `\`\`\`\n${body}\n\`\`\`` + } + + default: + return content + } +} + +function getSourceFilePath(snippet, repoRoot) { + if (snippet.sourceFilepath) { + return path.join(repoRoot, snippet.sourceFilepath) + } else { + throw new Error( + `Snippet "${snippet.snippetName}" has no source file path specified` + ) + } +} + +function processSnippet(snippet, paths, globalOptions = {}) { + const { repoRoot, outputFolderPath, verbose } = paths + try { + if (verbose) { + console.log(`Processing snippet: ${snippet.snippetName}`) + } + + if (!snippet.snippetName) { + throw new Error('Snippet missing required field: snippetName') + } + + if (!snippet.location) { + throw new Error( + `Snippet "${snippet.snippetName}" missing required field: location` + ) + } + + const sourceFilePath = getSourceFilePath(snippet, repoRoot) + + const fileContent = readFileContent(sourceFilePath) + + const extractedContent = extractSnippetContent( + fileContent, + snippet.location + ) + const skipTransform = + snippet.options && + (snippet.options.transform === 'rstjson' || + snippet.options.transform === 'rstinclude') + const normalizedContent = skipTransform + ? extractedContent + : applyIndentOption(extractedContent, resolveNormalizeIndent(snippet)) + + const substitutedContent = applyUrlSubstitutions( + normalizedContent, + globalOptions.urlSubstitutions, + snippet.options && snippet.options.urlSubstitutions + ) + + const formattedContent = formatSnippetContent( + substitutedContent, + snippet.options || {}, + globalOptions + ) + + const outputFileName = `${snippet.snippetName}.mdx` + const outputPath = path.join(outputFolderPath, outputFileName) + const outputPathDir = path.dirname(outputPath) + + fs.mkdirSync(outputPathDir, { recursive: true }) + + fs.writeFileSync(outputPath, formattedContent, 'utf8') + + if (verbose) { + console.log(`✓ Successfully extracted snippet to: ${outputPath}`) + } + } catch (error) { + console.error( + `✗ Error processing snippet "${snippet.snippetName}": ${error.message}` + ) + throw error + } +} + +/** + * Main function + * Reads docs/config/exportConfig.json and processes each snippet. + */ +function main() { + try { + const paths = parseCli(process.argv.slice(2)) + const configContent = readFileContent(paths.exportConfigPath) + const config = JSON.parse(configContent) + + if (!config.snippets || !Array.isArray(config.snippets)) { + throw new Error( + 'exportConfig.json must have a top-level "snippets" array' + ) + } + + let successCount = 0 + let errorCount = 0 + + const globalOptions = { + rstIncludeRefTargets: config.rstIncludeRefTargets || {}, + urlSubstitutions: config.urlSubstitutions || {}, + } + + for (const snippet of config.snippets) { + try { + processSnippet(snippet, paths, globalOptions) + successCount++ + } catch (error) { + errorCount++ + } + } + + console.log( + `\nProcessing complete: ${successCount} succeeded, ${errorCount} failed` + ) + + if (errorCount > 0) { + process.exit(1) + } + } catch (error) { + console.error(`Fatal error: ${error.message}`) + process.exit(1) + } +} + +main()