From 8c24dd80f26e0d02f497efd819388bbf69b30cc7 Mon Sep 17 00:00:00 2001 From: szymonrybczak Date: Wed, 23 Apr 2025 18:18:54 +0200 Subject: [PATCH 1/9] feat: add script for generating `llms.txt` file --- .gitignore | 6 +- scripts/generate-llms-txt.js | 265 +++++++++++++++++++++++++++++++++++ website/package.json | 2 +- 3 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 scripts/generate-llms-txt.js diff --git a/.gitignore b/.gitignore index 3ac1bf740a6..740757303ec 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,8 @@ website/build/ !.yarn/plugins !.yarn/releases !.yarn/sdks -!.yarn/versions \ No newline at end of file +!.yarn/versions + + +# Generated file(s) for llms +llms.txt diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js new file mode 100644 index 00000000000..94fbbfffadb --- /dev/null +++ b/scripts/generate-llms-txt.js @@ -0,0 +1,265 @@ +const fs = require('fs'); +const readline = require('readline'); +const https = require('https'); +const url = require('url'); +const path = require('path'); + +const OUTPUT_FILENAME = 'llms.txt'; +const TITLE = 'React Native Documentation'; +const DESCRIPTION = + 'React Native is a framework for building native apps using React. It lets you create mobile apps using only JavaScript and React.'; +const URL_PREFIX = 'https://reactnative.dev/docs/'; +const DOCS_PATH = '../docs/'; + +// Function to convert the TypeScript sidebar config to JSON +function convertSidebarConfigToJson(filePath) { + const fileContent = fs.readFileSync(filePath, 'utf8'); + + // Simple regex to extract the object between curly braces + const exportPattern = + /export default\s*({[\s\S]*?})\s*satisfies\s*SidebarsConfig;/; + const match = fileContent.match(exportPattern); + + if (!match) { + console.error('Could not find the sidebar configuration object'); + return null; + } + + let configText = match[1]; + + // Manual transformation approach + try { + // First, convert the TypeScript object to a JavaScript object using Function constructor + const tempFn = new Function(`return ${configText}`); + const jsObject = tempFn(); + return jsObject; + } catch (error) { + console.error('Error evaluating the configuration:', error); + return null; + } +} + +// Function to extract URLs from sidebar config +function extractUrlsFromSidebar(sidebarConfig) { + const urls = []; + + // Process each section (docs, api, components) + Object.entries(sidebarConfig).forEach(([section, categories]) => { + Object.entries(categories).forEach(([categoryName, items]) => { + processItemsForUrls(items, urls); + }); + }); + + return urls; +} + +// Recursive function to process items and extract URLs +function processItemsForUrls(items, urls) { + items.forEach(item => { + if (typeof item === 'string') { + urls.push(`${URL_PREFIX}${item}`); + } else if (typeof item === 'object') { + if (item.type === 'doc' && item.id) { + urls.push(`${URL_PREFIX}${item.id}`); + } else if (item.type === 'category' && Array.isArray(item.items)) { + processItemsForUrls(item.items, urls); + } + } + }); +} + +// Function to check URL status +function checkUrl(urlString) { + return new Promise(resolve => { + const parsedUrl = url.parse(urlString); + + const options = { + hostname: parsedUrl.hostname, + path: parsedUrl.path, + method: 'HEAD', + timeout: 5000, + }; + + const req = https.request(options, res => { + resolve({ + url: urlString, + status: res.statusCode, + is404: res.statusCode === 404, + }); + }); + + req.on('error', error => { + resolve({ + url: urlString, + status: 'Error', + is404: false, + error: error.message, + }); + }); + + req.on('timeout', () => { + req.destroy(); + resolve({ + url: urlString, + status: 'Timeout', + is404: false, + }); + }); + + req.end(); + }); +} + +// Process each URL +async function processUrls(urls) { + const unavailableUrls = []; + + for (const urlToCheck of urls) { + const result = await checkUrl(urlToCheck); + if ( + result.is404 || + result.status === 'Error' || + result.status === 'Timeout' + ) { + unavailableUrls.push({ + url: urlToCheck, + status: result.status, + error: result.error || null, + }); + } + } + + const result = { + totalUrls: urls.length, + unavailableUrls: unavailableUrls, + }; + + if (unavailableUrls.length > 0) { + console.log(JSON.stringify(result, null, 2)); + } else { + console.log(JSON.stringify(result, null, 2)); + } + + return result; +} + +// Function to extract title from markdown frontmatter +function extractTitleFromMarkdown(filePath) { + try { + const content = fs.readFileSync(filePath, 'utf8'); + const frontmatterMatch = content.match(/---\n([\s\S]*?)\n---/); + if (frontmatterMatch) { + const frontmatter = frontmatterMatch[1]; + const titleMatch = frontmatter.match(/title:\s*(.*)/); + if (titleMatch) { + return titleMatch[1].trim(); + } + } + // If no title found, use the filename + return filePath.split('/').pop().replace('.md', ''); + } catch (error) { + console.error(`Error reading file ${filePath}:`, error); + return filePath.split('/').pop().replace('.md', ''); + } +} + +// Function to map special cases for file names that don't match the sidebar +function mapDocPath(item) { + const specialCases = { + 'environment-setup': 'getting-started.md', + 'native-platform': 'native-platforms.md', + 'turbo-native-modules-introduction': 'turbo-native-modules.md', + 'fabric-native-components-introduction': 'fabric-native-components.md', + }; + + if (typeof item === 'string') { + return specialCases[item] || `${item}.md`; + } else if (item.type === 'doc' && item.id) { + return specialCases[item.id] || `${item.id}.md`; + } + return `${item}.md`; +} + +// Function to generate markdown documentation +function generateMarkdown(sidebarConfig) { + let markdown = `# ${TITLE}\n\n`; + markdown += `> ${DESCRIPTION}\n\n`; + markdown += `This documentation covers all aspects of using React Native, from installation to advanced usage.\n\n`; + + // Process each section (docs, api, components) + Object.entries(sidebarConfig).forEach(([section, categories]) => { + markdown += `## ${section.charAt(0).toUpperCase() + section.slice(1)}\n\n`; + + // Process each category within the section + Object.entries(categories).forEach(([categoryName, items]) => { + markdown += `### ${categoryName}\n\n`; + + // Process each item in the category + items.forEach(item => { + if (typeof item === 'string') { + // This is a direct page reference + const docPath = `${DOCS_PATH}${mapDocPath(item)}`; + const title = extractTitleFromMarkdown(docPath); + markdown += `- [${title}](${URL_PREFIX}${item})\n`; + } else if (typeof item === 'object') { + if (item.type === 'doc' && item.id) { + // This is a doc reference with an explicit ID + const docPath = `${DOCS_PATH}${mapDocPath(item)}`; + const title = extractTitleFromMarkdown(docPath); + markdown += `- [${title}](${URL_PREFIX}${item.id})\n`; + } else if (item.type === 'category' && Array.isArray(item.items)) { + // This is a category with nested items + markdown += `#### ${item.label}\n\n`; + item.items.forEach(nestedItem => { + if (typeof nestedItem === 'string') { + const docPath = `${DOCS_PATH}${mapDocPath(nestedItem)}`; + const title = extractTitleFromMarkdown(docPath); + markdown += `- [${title}](${URL_PREFIX}${nestedItem})\n`; + } else if (nestedItem.type === 'doc' && nestedItem.id) { + const docPath = `${DOCS_PATH}${mapDocPath(nestedItem)}`; + const title = extractTitleFromMarkdown(docPath); + markdown += `- [${title}](${URL_PREFIX}${nestedItem.id})\n`; + } + }); + } + } + }); + }); + }); + + // Add newlines after all markdown headers + markdown = markdown.replace(/(#+ .*)\n/g, '\n$1\n'); + + return markdown; +} + +const inputFilePath = './sidebars.ts'; +const outputFilePath = inputFilePath.replace(/\.tsx?$/, '-urls.txt'); + +const sidebarConfig = convertSidebarConfigToJson(inputFilePath); +if (sidebarConfig) { + const urls = extractUrlsFromSidebar(sidebarConfig); + + // First check URLs for 404 errors + processUrls(urls) + .then(result => { + if (result.unavailableUrls.length === 0) { + // Only generate documentation if all URLs are valid + const markdown = generateMarkdown(sidebarConfig); + fs.writeFileSync(path.join('static/', OUTPUT_FILENAME), markdown); + console.log( + `Successfully generated documentation to: ${OUTPUT_FILENAME}` + ); + } else { + console.error('Documentation generation skipped due to broken links'); + process.exit(1); + } + }) + .catch(err => { + console.error('Error processing URLs:', err); + process.exit(1); + }); +} else { + console.error('Failed to convert sidebar config to JSON'); + process.exit(1); +} diff --git a/website/package.json b/website/package.json index b4806be7542..26a9b3aaab9 100644 --- a/website/package.json +++ b/website/package.json @@ -12,7 +12,7 @@ "scripts": { "docusaurus": "docusaurus", "start": "docusaurus start", - "build": "docusaurus build && yarn run update-redirect ./build/_redirects ./versions.json", + "build": "docusaurus build && yarn run update-redirect ./build/_redirects ./versions.json && node ../scripts/generate-llms-txt.js", "build:fast": "PREVIEW_DEPLOY=true yarn run build", "tsc": "npx tsc --noEmit", "swizzle": "docusaurus swizzle", From abfafd024b4802f2c92e4fe1bf5eab6e18aa9343 Mon Sep 17 00:00:00 2001 From: szymonrybczak Date: Wed, 23 Apr 2025 18:31:29 +0200 Subject: [PATCH 2/9] fix: save file in `build/` dir --- scripts/generate-llms-txt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index 94fbbfffadb..a84c9b6d03c 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -246,7 +246,7 @@ if (sidebarConfig) { if (result.unavailableUrls.length === 0) { // Only generate documentation if all URLs are valid const markdown = generateMarkdown(sidebarConfig); - fs.writeFileSync(path.join('static/', OUTPUT_FILENAME), markdown); + fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), markdown); console.log( `Successfully generated documentation to: ${OUTPUT_FILENAME}` ); From 3b67151244d68e924adb465dfb725b2543928ec0 Mon Sep 17 00:00:00 2001 From: Szymon Rybczak Date: Thu, 24 Apr 2025 14:26:35 +0200 Subject: [PATCH 3/9] Update scripts/generate-llms-txt.js Co-authored-by: Bartosz Kaszubowski --- scripts/generate-llms-txt.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index a84c9b6d03c..08d8c4fe502 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -1,5 +1,4 @@ const fs = require('fs'); -const readline = require('readline'); const https = require('https'); const url = require('url'); const path = require('path'); From 4944e880fc4ff2265ea4f13ad0420618b7d20a64 Mon Sep 17 00:00:00 2001 From: Szymon Rybczak Date: Thu, 24 Apr 2025 14:26:50 +0200 Subject: [PATCH 4/9] Update scripts/generate-llms-txt.js Co-authored-by: Bartosz Kaszubowski --- scripts/generate-llms-txt.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index 08d8c4fe502..850b15f9de6 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -43,8 +43,8 @@ function extractUrlsFromSidebar(sidebarConfig) { const urls = []; // Process each section (docs, api, components) - Object.entries(sidebarConfig).forEach(([section, categories]) => { - Object.entries(categories).forEach(([categoryName, items]) => { + Object.entries(sidebarConfig).forEach(([_, categories]) => { + Object.entries(categories).forEach(([_, items]) => { processItemsForUrls(items, urls); }); }); From f750e6e9615bbada7ee97d41f19980c0b514433f Mon Sep 17 00:00:00 2001 From: Szymon Rybczak Date: Thu, 24 Apr 2025 14:27:56 +0200 Subject: [PATCH 5/9] Update scripts/generate-llms-txt.js Co-authored-by: Bartosz Kaszubowski --- scripts/generate-llms-txt.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index 850b15f9de6..561dc074391 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -226,10 +226,8 @@ function generateMarkdown(sidebarConfig) { }); }); - // Add newlines after all markdown headers - markdown = markdown.replace(/(#+ .*)\n/g, '\n$1\n'); - - return markdown; + // Format and cleanup whitespaces + return markdown.replace(/(#+ .*)\n/g, '\n$1\n').replace(/\n(\n)+/g, '\n\n'); } const inputFilePath = './sidebars.ts'; From 7e0b7c031847128f36ba9729c45ca8be3fd05cce Mon Sep 17 00:00:00 2001 From: szymonrybczak Date: Mon, 28 Apr 2025 12:22:22 +0200 Subject: [PATCH 6/9] chore: move script --- website/package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/package.json b/website/package.json index 26a9b3aaab9..35a9a42ddc3 100644 --- a/website/package.json +++ b/website/package.json @@ -12,7 +12,7 @@ "scripts": { "docusaurus": "docusaurus", "start": "docusaurus start", - "build": "docusaurus build && yarn run update-redirect ./build/_redirects ./versions.json && node ../scripts/generate-llms-txt.js", + "build": "docusaurus build && yarn run update-redirect ./build/_redirects ./versions.json && yarn run generate-llms-txt", "build:fast": "PREVIEW_DEPLOY=true yarn run build", "tsc": "npx tsc --noEmit", "swizzle": "docusaurus swizzle", @@ -35,7 +35,8 @@ "language:lint": "cd ../ && alex && case-police 'docs/*.md' -d ./website/react-native-dict.json --disable SDK,URI", "language:lint:versioned": "cd ../ && alex . && case-police '**/*.md' -d ./website/react-native-dict.json --disable SDK,URI", "ci:lint": "yarn lint && yarn lint:examples && yarn lint:versioned && yarn language:lint:versioned && yarn lint:markdown && yarn lint:format", - "pwa:generate": "npx pwa-asset-generator ./static/img/header_logo.svg ./static/img/pwa --padding '40px' --background 'rgb(32, 35, 42)' --icon-only --opaque true" + "pwa:generate": "npx pwa-asset-generator ./static/img/header_logo.svg ./static/img/pwa --padding '40px' --background 'rgb(32, 35, 42)' --icon-only --opaque true", + "generate-llms-txt": "node ../scripts/generate-llms-txt.js" }, "browserslist": { "production": [ From 3ebf6f5841cfe30175f4a252eba6668a5bbc3951 Mon Sep 17 00:00:00 2001 From: szymonrybczak Date: Mon, 28 Apr 2025 12:24:49 +0200 Subject: [PATCH 7/9] fix: use TypeScript programmatically --- scripts/generate-llms-txt.js | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index 561dc074391..e96d3f2ddd9 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -2,6 +2,7 @@ const fs = require('fs'); const https = require('https'); const url = require('url'); const path = require('path'); +const ts = require('typescript'); const OUTPUT_FILENAME = 'llms.txt'; const TITLE = 'React Native Documentation'; @@ -12,29 +13,28 @@ const DOCS_PATH = '../docs/'; // Function to convert the TypeScript sidebar config to JSON function convertSidebarConfigToJson(filePath) { - const fileContent = fs.readFileSync(filePath, 'utf8'); + const inputFileContent = fs.readFileSync(filePath, 'utf8'); + const tempFilePath = path.join(__dirname, 'temp-sidebar.js'); - // Simple regex to extract the object between curly braces - const exportPattern = - /export default\s*({[\s\S]*?})\s*satisfies\s*SidebarsConfig;/; - const match = fileContent.match(exportPattern); - - if (!match) { - console.error('Could not find the sidebar configuration object'); - return null; - } + try { + const {outputText} = ts.transpileModule(inputFileContent, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2015, + }, + }); - let configText = match[1]; + fs.writeFileSync(tempFilePath, outputText); - // Manual transformation approach - try { - // First, convert the TypeScript object to a JavaScript object using Function constructor - const tempFn = new Function(`return ${configText}`); - const jsObject = tempFn(); - return jsObject; + const sidebarModule = require(tempFilePath); + return sidebarModule.default; } catch (error) { - console.error('Error evaluating the configuration:', error); + console.error('Error converting sidebar config:', error); return null; + } finally { + if (fs.existsSync(tempFilePath)) { + fs.unlinkSync(tempFilePath); + } } } From 6f3dce3ce224833747320e28eee96cc4d1da5609 Mon Sep 17 00:00:00 2001 From: szymonrybczak Date: Mon, 28 Apr 2025 23:41:01 +0200 Subject: [PATCH 8/9] feat: gather data from all sections of docs --- scripts/generate-llms-txt.js | 248 +++++++++++++++++++++++++---------- 1 file changed, 180 insertions(+), 68 deletions(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index e96d3f2ddd9..fbf92a84a8d 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -8,8 +8,7 @@ const OUTPUT_FILENAME = 'llms.txt'; const TITLE = 'React Native Documentation'; const DESCRIPTION = 'React Native is a framework for building native apps using React. It lets you create mobile apps using only JavaScript and React.'; -const URL_PREFIX = 'https://reactnative.dev/docs/'; -const DOCS_PATH = '../docs/'; +const URL_PREFIX = 'https://reactnative.dev'; // Function to convert the TypeScript sidebar config to JSON function convertSidebarConfigToJson(filePath) { @@ -26,7 +25,11 @@ function convertSidebarConfigToJson(filePath) { fs.writeFileSync(tempFilePath, outputText); + // Clear require cache for the temp file + delete require.cache[require.resolve(tempFilePath)]; + const sidebarModule = require(tempFilePath); + return sidebarModule.default; } catch (error) { console.error('Error converting sidebar config:', error); @@ -38,33 +41,55 @@ function convertSidebarConfigToJson(filePath) { } } +const SLUG_TO_URL = { + 'architecture-overview': 'overview', + 'architecture-glossary': 'glossary', +}; + // Function to extract URLs from sidebar config -function extractUrlsFromSidebar(sidebarConfig) { +function extractUrlsFromSidebar(sidebarConfig, prefix) { const urls = []; // Process each section (docs, api, components) Object.entries(sidebarConfig).forEach(([_, categories]) => { Object.entries(categories).forEach(([_, items]) => { - processItemsForUrls(items, urls); + processItemsForUrls(items, urls, prefix); }); }); + // Replace slugs with their mapped URLs + urls.forEach((url, index) => { + for (const [slug, mappedUrl] of Object.entries(SLUG_TO_URL)) { + if (url.includes(slug)) { + urls[index] = url.replace(slug, mappedUrl); + break; + } + } + }); + return urls; } // Recursive function to process items and extract URLs -function processItemsForUrls(items, urls) { - items.forEach(item => { - if (typeof item === 'string') { - urls.push(`${URL_PREFIX}${item}`); - } else if (typeof item === 'object') { - if (item.type === 'doc' && item.id) { - urls.push(`${URL_PREFIX}${item.id}`); - } else if (item.type === 'category' && Array.isArray(item.items)) { - processItemsForUrls(item.items, urls); +function processItemsForUrls(items, urls, prefix) { + if (typeof items === 'object' && Array.isArray(items.items)) { + processItemsForUrls(items.items, urls, prefix); + return; + } + + if (Array.isArray(items)) { + items.forEach(item => { + if (typeof item === 'string') { + urls.push(`${URL_PREFIX}${prefix}/${item}`); + } else if (typeof item === 'object') { + if (item.type === 'doc' && item.id) { + urls.push(`${URL_PREFIX}${prefix}/${item.id}`); + } else if (item.type === 'category' && Array.isArray(item.items)) { + processItemsForUrls(item.items, urls, prefix); + } } - } - }); + }); + } } // Function to check URL status @@ -143,27 +168,38 @@ async function processUrls(urls) { } // Function to extract title from markdown frontmatter -function extractTitleFromMarkdown(filePath) { +function extractMetadataFromMarkdown(filePath) { try { const content = fs.readFileSync(filePath, 'utf8'); const frontmatterMatch = content.match(/---\n([\s\S]*?)\n---/); if (frontmatterMatch) { const frontmatter = frontmatterMatch[1]; const titleMatch = frontmatter.match(/title:\s*(.*)/); - if (titleMatch) { - return titleMatch[1].trim(); - } + const slugMatch = frontmatter.match(/slug:\s*(.*)/); + + return { + title: titleMatch + ? titleMatch[1].trim() + : filePath.split('/').pop().replace('.md', ''), + slug: slugMatch ? slugMatch[1].trim().replace(/^\//, '') : null, + }; } - // If no title found, use the filename - return filePath.split('/').pop().replace('.md', ''); + // If no frontmatter found, use the filename + return { + title: filePath.split('/').pop().replace('.md', ''), + slug: null, + }; } catch (error) { console.error(`Error reading file ${filePath}:`, error); - return filePath.split('/').pop().replace('.md', ''); + return { + title: filePath.split('/').pop().replace('.md', ''), + slug: null, + }; } } // Function to map special cases for file names that don't match the sidebar -function mapDocPath(item) { +function mapDocPath(item, prefix) { const specialCases = { 'environment-setup': 'getting-started.md', 'native-platform': 'native-platforms.md', @@ -171,6 +207,10 @@ function mapDocPath(item) { 'fabric-native-components-introduction': 'fabric-native-components.md', }; + if (prefix === '/contributing') { + specialCases['overview'] = 'contributing-overview.md'; + } + if (typeof item === 'string') { return specialCases[item] || `${item}.md`; } else if (item.type === 'doc' && item.id) { @@ -179,11 +219,9 @@ function mapDocPath(item) { return `${item}.md`; } -// Function to generate markdown documentation -function generateMarkdown(sidebarConfig) { - let markdown = `# ${TITLE}\n\n`; - markdown += `> ${DESCRIPTION}\n\n`; - markdown += `This documentation covers all aspects of using React Native, from installation to advanced usage.\n\n`; +// Function to generate output for each sidebar +function generateMarkdown(sidebarConfig, docPath, prefix) { + let markdown = ''; // Process each section (docs, api, components) Object.entries(sidebarConfig).forEach(([section, categories]) => { @@ -191,33 +229,46 @@ function generateMarkdown(sidebarConfig) { // Process each category within the section Object.entries(categories).forEach(([categoryName, items]) => { - markdown += `### ${categoryName}\n\n`; + markdown += `### ${categoryName === '0' ? 'General' : categoryName}\n\n`; + + if (typeof items === 'object' && Array.isArray(items.items)) { + items = items.items; + } + const reorderedArray = items.every(item => typeof item === 'string') + ? items + : [...items].sort((a, b) => + typeof a === 'string' && typeof b !== 'string' + ? -1 + : typeof a !== 'string' && typeof b === 'string' + ? 1 + : 0 + ); // Process each item in the category - items.forEach(item => { + reorderedArray.forEach(item => { if (typeof item === 'string') { // This is a direct page reference - const docPath = `${DOCS_PATH}${mapDocPath(item)}`; - const title = extractTitleFromMarkdown(docPath); - markdown += `- [${title}](${URL_PREFIX}${item})\n`; + const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item})\n`; } else if (typeof item === 'object') { if (item.type === 'doc' && item.id) { // This is a doc reference with an explicit ID - const docPath = `${DOCS_PATH}${mapDocPath(item)}`; - const title = extractTitleFromMarkdown(docPath); - markdown += `- [${title}](${URL_PREFIX}${item.id})\n`; + const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item.id})\n`; } else if (item.type === 'category' && Array.isArray(item.items)) { // This is a category with nested items markdown += `#### ${item.label}\n\n`; item.items.forEach(nestedItem => { if (typeof nestedItem === 'string') { - const docPath = `${DOCS_PATH}${mapDocPath(nestedItem)}`; - const title = extractTitleFromMarkdown(docPath); - markdown += `- [${title}](${URL_PREFIX}${nestedItem})\n`; + const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem})\n`; } else if (nestedItem.type === 'doc' && nestedItem.id) { - const docPath = `${DOCS_PATH}${mapDocPath(nestedItem)}`; - const title = extractTitleFromMarkdown(docPath); - markdown += `- [${title}](${URL_PREFIX}${nestedItem.id})\n`; + const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem.id})\n`; } }); } @@ -230,33 +281,94 @@ function generateMarkdown(sidebarConfig) { return markdown.replace(/(#+ .*)\n/g, '\n$1\n').replace(/\n(\n)+/g, '\n\n'); } -const inputFilePath = './sidebars.ts'; -const outputFilePath = inputFilePath.replace(/\.tsx?$/, '-urls.txt'); - -const sidebarConfig = convertSidebarConfigToJson(inputFilePath); -if (sidebarConfig) { - const urls = extractUrlsFromSidebar(sidebarConfig); - - // First check URLs for 404 errors - processUrls(urls) - .then(result => { - if (result.unavailableUrls.length === 0) { - // Only generate documentation if all URLs are valid - const markdown = generateMarkdown(sidebarConfig); - fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), markdown); - console.log( - `Successfully generated documentation to: ${OUTPUT_FILENAME}` - ); - } else { - console.error('Documentation generation skipped due to broken links'); - process.exit(1); - } +const inputFilePaths = [ + { + name: 'sidebars.ts', + docPath: '../docs/', + prefix: '/docs', + }, + { + name: 'sidebarsArchitecture.ts', + docPath: './architecture/', + prefix: '/architecture', + }, + { + name: 'sidebarsCommunity.ts', + docPath: './community/', + prefix: '/community', + }, + { + name: 'sidebarsContributing.ts', + docPath: './contributing/', + prefix: '/contributing', + }, +]; + +let output = `# ${TITLE}\n\n`; +output += `> ${DESCRIPTION}\n\n`; +output += `This documentation covers all aspects of using React Native, from installation to advanced usage.\n\n`; + +const generateOutput = () => { + const results = []; + const promises = []; + + for (const {name, docPath, prefix} of inputFilePaths) { + const inputFilePath = `./${name}`; + const outputFilePath = inputFilePath.replace(/\.tsx?$/, '-urls.txt'); + + const sidebarConfig = convertSidebarConfigToJson(inputFilePath); + if (sidebarConfig) { + const urls = extractUrlsFromSidebar(sidebarConfig, prefix); + + // First check URLs for 404 errors + const promise = processUrls(urls) + .then(result => { + if (result.unavailableUrls.length === 0) { + // Only generate documentation if all URLs are valid + const markdown = generateMarkdown(sidebarConfig, docPath, prefix); + results.push({ markdown, prefix }); + console.log(`Successfully generated output from ${inputFilePath}`); + } else { + console.error( + 'Documentation generation skipped due to broken links' + ); + process.exit(1); + } + }) + .catch(err => { + console.error('Error processing URLs:', err); + process.exit(1); + }); + + promises.push(promise); + } else { + console.error('Failed to convert sidebar config to JSON'); + process.exit(1); + } + } + + // Wait for all promises to complete before writing the file + Promise.all(promises) + .then(() => { + // Sort results to ensure docs section is first + results.sort((a, b) => { + if (a.prefix === '/docs') return -1; + if (b.prefix === '/docs') return 1; + return 0; + }); + + // Combine all markdown content in the correct order + output += results.map(r => r.markdown).join('\n'); + + fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), output); + console.log( + `Successfully generated documentation to: ${OUTPUT_FILENAME}` + ); }) .catch(err => { - console.error('Error processing URLs:', err); + console.error('Error during processing:', err); process.exit(1); }); -} else { - console.error('Failed to convert sidebar config to JSON'); - process.exit(1); -} +}; + +generateOutput(); From 61232596c9caca759c822829b694d33bce5ad641 Mon Sep 17 00:00:00 2001 From: szymonrybczak Date: Tue, 29 Apr 2025 19:22:47 +0200 Subject: [PATCH 9/9] chore: apply code review improvements --- scripts/generate-llms-txt.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js index fbf92a84a8d..3526afb9eb8 100644 --- a/scripts/generate-llms-txt.js +++ b/scripts/generate-llms-txt.js @@ -314,7 +314,6 @@ const generateOutput = () => { for (const {name, docPath, prefix} of inputFilePaths) { const inputFilePath = `./${name}`; - const outputFilePath = inputFilePath.replace(/\.tsx?$/, '-urls.txt'); const sidebarConfig = convertSidebarConfigToJson(inputFilePath); if (sidebarConfig) { @@ -326,7 +325,7 @@ const generateOutput = () => { if (result.unavailableUrls.length === 0) { // Only generate documentation if all URLs are valid const markdown = generateMarkdown(sidebarConfig, docPath, prefix); - results.push({ markdown, prefix }); + results.push({markdown, prefix}); console.log(`Successfully generated output from ${inputFilePath}`); } else { console.error( @@ -359,7 +358,7 @@ const generateOutput = () => { // Combine all markdown content in the correct order output += results.map(r => r.markdown).join('\n'); - + fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), output); console.log( `Successfully generated documentation to: ${OUTPUT_FILENAME}`