diff --git a/.gitignore b/.gitignore index 3ac1bf740a6..740757303ec 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,8 @@ website/build/ !.yarn/plugins !.yarn/releases !.yarn/sdks -!.yarn/versions \ No newline at end of file +!.yarn/versions + + +# Generated file(s) for llms +llms.txt diff --git a/scripts/generate-llms-txt.js b/scripts/generate-llms-txt.js new file mode 100644 index 00000000000..3526afb9eb8 --- /dev/null +++ b/scripts/generate-llms-txt.js @@ -0,0 +1,373 @@ +const fs = require('fs'); +const https = require('https'); +const url = require('url'); +const path = require('path'); +const ts = require('typescript'); + +const OUTPUT_FILENAME = 'llms.txt'; +const TITLE = 'React Native Documentation'; +const DESCRIPTION = + 'React Native is a framework for building native apps using React. It lets you create mobile apps using only JavaScript and React.'; +const URL_PREFIX = 'https://reactnative.dev'; + +// Function to convert the TypeScript sidebar config to JSON +function convertSidebarConfigToJson(filePath) { + const inputFileContent = fs.readFileSync(filePath, 'utf8'); + const tempFilePath = path.join(__dirname, 'temp-sidebar.js'); + + try { + const {outputText} = ts.transpileModule(inputFileContent, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2015, + }, + }); + + fs.writeFileSync(tempFilePath, outputText); + + // Clear require cache for the temp file + delete require.cache[require.resolve(tempFilePath)]; + + const sidebarModule = require(tempFilePath); + + return sidebarModule.default; + } catch (error) { + console.error('Error converting sidebar config:', error); + return null; + } finally { + if (fs.existsSync(tempFilePath)) { + fs.unlinkSync(tempFilePath); + } + } +} + +const SLUG_TO_URL = { + 'architecture-overview': 'overview', + 'architecture-glossary': 'glossary', +}; + +// Function to extract URLs from sidebar config +function extractUrlsFromSidebar(sidebarConfig, prefix) { + const urls = []; + + // Process each section (docs, api, components) + Object.entries(sidebarConfig).forEach(([_, categories]) => { + Object.entries(categories).forEach(([_, items]) => { + processItemsForUrls(items, urls, prefix); + }); + }); + + // Replace slugs with their mapped URLs + urls.forEach((url, index) => { + for (const [slug, mappedUrl] of Object.entries(SLUG_TO_URL)) { + if (url.includes(slug)) { + urls[index] = url.replace(slug, mappedUrl); + break; + } + } + }); + + return urls; +} + +// Recursive function to process items and extract URLs +function processItemsForUrls(items, urls, prefix) { + if (typeof items === 'object' && Array.isArray(items.items)) { + processItemsForUrls(items.items, urls, prefix); + return; + } + + if (Array.isArray(items)) { + items.forEach(item => { + if (typeof item === 'string') { + urls.push(`${URL_PREFIX}${prefix}/${item}`); + } else if (typeof item === 'object') { + if (item.type === 'doc' && item.id) { + urls.push(`${URL_PREFIX}${prefix}/${item.id}`); + } else if (item.type === 'category' && Array.isArray(item.items)) { + processItemsForUrls(item.items, urls, prefix); + } + } + }); + } +} + +// Function to check URL status +function checkUrl(urlString) { + return new Promise(resolve => { + const parsedUrl = url.parse(urlString); + + const options = { + hostname: parsedUrl.hostname, + path: parsedUrl.path, + method: 'HEAD', + timeout: 5000, + }; + + const req = https.request(options, res => { + resolve({ + url: urlString, + status: res.statusCode, + is404: res.statusCode === 404, + }); + }); + + req.on('error', error => { + resolve({ + url: urlString, + status: 'Error', + is404: false, + error: error.message, + }); + }); + + req.on('timeout', () => { + req.destroy(); + resolve({ + url: urlString, + status: 'Timeout', + is404: false, + }); + }); + + req.end(); + }); +} + +// Process each URL +async function processUrls(urls) { + const unavailableUrls = []; + + for (const urlToCheck of urls) { + const result = await checkUrl(urlToCheck); + if ( + result.is404 || + result.status === 'Error' || + result.status === 'Timeout' + ) { + unavailableUrls.push({ + url: urlToCheck, + status: result.status, + error: result.error || null, + }); + } + } + + const result = { + totalUrls: urls.length, + unavailableUrls: unavailableUrls, + }; + + if (unavailableUrls.length > 0) { + console.log(JSON.stringify(result, null, 2)); + } else { + console.log(JSON.stringify(result, null, 2)); + } + + return result; +} + +// Function to extract title from markdown frontmatter +function extractMetadataFromMarkdown(filePath) { + try { + const content = fs.readFileSync(filePath, 'utf8'); + const frontmatterMatch = content.match(/---\n([\s\S]*?)\n---/); + if (frontmatterMatch) { + const frontmatter = frontmatterMatch[1]; + const titleMatch = frontmatter.match(/title:\s*(.*)/); + const slugMatch = frontmatter.match(/slug:\s*(.*)/); + + return { + title: titleMatch + ? titleMatch[1].trim() + : filePath.split('/').pop().replace('.md', ''), + slug: slugMatch ? slugMatch[1].trim().replace(/^\//, '') : null, + }; + } + // If no frontmatter found, use the filename + return { + title: filePath.split('/').pop().replace('.md', ''), + slug: null, + }; + } catch (error) { + console.error(`Error reading file ${filePath}:`, error); + return { + title: filePath.split('/').pop().replace('.md', ''), + slug: null, + }; + } +} + +// Function to map special cases for file names that don't match the sidebar +function mapDocPath(item, prefix) { + const specialCases = { + 'environment-setup': 'getting-started.md', + 'native-platform': 'native-platforms.md', + 'turbo-native-modules-introduction': 'turbo-native-modules.md', + 'fabric-native-components-introduction': 'fabric-native-components.md', + }; + + if (prefix === '/contributing') { + specialCases['overview'] = 'contributing-overview.md'; + } + + if (typeof item === 'string') { + return specialCases[item] || `${item}.md`; + } else if (item.type === 'doc' && item.id) { + return specialCases[item.id] || `${item.id}.md`; + } + return `${item}.md`; +} + +// Function to generate output for each sidebar +function generateMarkdown(sidebarConfig, docPath, prefix) { + let markdown = ''; + + // Process each section (docs, api, components) + Object.entries(sidebarConfig).forEach(([section, categories]) => { + markdown += `## ${section.charAt(0).toUpperCase() + section.slice(1)}\n\n`; + + // Process each category within the section + Object.entries(categories).forEach(([categoryName, items]) => { + markdown += `### ${categoryName === '0' ? 'General' : categoryName}\n\n`; + + if (typeof items === 'object' && Array.isArray(items.items)) { + items = items.items; + } + const reorderedArray = items.every(item => typeof item === 'string') + ? items + : [...items].sort((a, b) => + typeof a === 'string' && typeof b !== 'string' + ? -1 + : typeof a !== 'string' && typeof b === 'string' + ? 1 + : 0 + ); + + // Process each item in the category + reorderedArray.forEach(item => { + if (typeof item === 'string') { + // This is a direct page reference + const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item})\n`; + } else if (typeof item === 'object') { + if (item.type === 'doc' && item.id) { + // This is a doc reference with an explicit ID + const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item.id})\n`; + } else if (item.type === 'category' && Array.isArray(item.items)) { + // This is a category with nested items + markdown += `#### ${item.label}\n\n`; + item.items.forEach(nestedItem => { + if (typeof nestedItem === 'string') { + const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem})\n`; + } else if (nestedItem.type === 'doc' && nestedItem.id) { + const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`; + const {title, slug} = extractMetadataFromMarkdown(fullDocPath); + markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem.id})\n`; + } + }); + } + } + }); + }); + }); + + // Format and cleanup whitespaces + return markdown.replace(/(#+ .*)\n/g, '\n$1\n').replace(/\n(\n)+/g, '\n\n'); +} + +const inputFilePaths = [ + { + name: 'sidebars.ts', + docPath: '../docs/', + prefix: '/docs', + }, + { + name: 'sidebarsArchitecture.ts', + docPath: './architecture/', + prefix: '/architecture', + }, + { + name: 'sidebarsCommunity.ts', + docPath: './community/', + prefix: '/community', + }, + { + name: 'sidebarsContributing.ts', + docPath: './contributing/', + prefix: '/contributing', + }, +]; + +let output = `# ${TITLE}\n\n`; +output += `> ${DESCRIPTION}\n\n`; +output += `This documentation covers all aspects of using React Native, from installation to advanced usage.\n\n`; + +const generateOutput = () => { + const results = []; + const promises = []; + + for (const {name, docPath, prefix} of inputFilePaths) { + const inputFilePath = `./${name}`; + + const sidebarConfig = convertSidebarConfigToJson(inputFilePath); + if (sidebarConfig) { + const urls = extractUrlsFromSidebar(sidebarConfig, prefix); + + // First check URLs for 404 errors + const promise = processUrls(urls) + .then(result => { + if (result.unavailableUrls.length === 0) { + // Only generate documentation if all URLs are valid + const markdown = generateMarkdown(sidebarConfig, docPath, prefix); + results.push({markdown, prefix}); + console.log(`Successfully generated output from ${inputFilePath}`); + } else { + console.error( + 'Documentation generation skipped due to broken links' + ); + process.exit(1); + } + }) + .catch(err => { + console.error('Error processing URLs:', err); + process.exit(1); + }); + + promises.push(promise); + } else { + console.error('Failed to convert sidebar config to JSON'); + process.exit(1); + } + } + + // Wait for all promises to complete before writing the file + Promise.all(promises) + .then(() => { + // Sort results to ensure docs section is first + results.sort((a, b) => { + if (a.prefix === '/docs') return -1; + if (b.prefix === '/docs') return 1; + return 0; + }); + + // Combine all markdown content in the correct order + output += results.map(r => r.markdown).join('\n'); + + fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), output); + console.log( + `Successfully generated documentation to: ${OUTPUT_FILENAME}` + ); + }) + .catch(err => { + console.error('Error during processing:', err); + process.exit(1); + }); +}; + +generateOutput(); diff --git a/website/package.json b/website/package.json index b4806be7542..35a9a42ddc3 100644 --- a/website/package.json +++ b/website/package.json @@ -12,7 +12,7 @@ "scripts": { "docusaurus": "docusaurus", "start": "docusaurus start", - "build": "docusaurus build && yarn run update-redirect ./build/_redirects ./versions.json", + "build": "docusaurus build && yarn run update-redirect ./build/_redirects ./versions.json && yarn run generate-llms-txt", "build:fast": "PREVIEW_DEPLOY=true yarn run build", "tsc": "npx tsc --noEmit", "swizzle": "docusaurus swizzle", @@ -35,7 +35,8 @@ "language:lint": "cd ../ && alex && case-police 'docs/*.md' -d ./website/react-native-dict.json --disable SDK,URI", "language:lint:versioned": "cd ../ && alex . && case-police '**/*.md' -d ./website/react-native-dict.json --disable SDK,URI", "ci:lint": "yarn lint && yarn lint:examples && yarn lint:versioned && yarn language:lint:versioned && yarn lint:markdown && yarn lint:format", - "pwa:generate": "npx pwa-asset-generator ./static/img/header_logo.svg ./static/img/pwa --padding '40px' --background 'rgb(32, 35, 42)' --icon-only --opaque true" + "pwa:generate": "npx pwa-asset-generator ./static/img/header_logo.svg ./static/img/pwa --padding '40px' --background 'rgb(32, 35, 42)' --icon-only --opaque true", + "generate-llms-txt": "node ../scripts/generate-llms-txt.js" }, "browserslist": { "production": [