Skip to content

Commit 6f3dce3

Browse files
szymonrybczakszymonrybczak
committed
feat: gather data from all sections of docs
1 parent 3ebf6f5 commit 6f3dce3

1 file changed

Lines changed: 180 additions & 68 deletions

File tree

‎scripts/generate-llms-txt.js‎

Lines changed: 180 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ const OUTPUT_FILENAME = 'llms.txt';
88
const TITLE = 'React Native Documentation';
99
const DESCRIPTION =
1010
'React Native is a framework for building native apps using React. It lets you create mobile apps using only JavaScript and React.';
11-
const URL_PREFIX = 'https://reactnative.dev/docs/';
12-
const DOCS_PATH = '../docs/';
11+
const URL_PREFIX = 'https://reactnative.dev';
1312

1413
// Function to convert the TypeScript sidebar config to JSON
1514
function convertSidebarConfigToJson(filePath) {
@@ -26,7 +25,11 @@ function convertSidebarConfigToJson(filePath) {
2625

2726
fs.writeFileSync(tempFilePath, outputText);
2827

28+
// Clear require cache for the temp file
29+
delete require.cache[require.resolve(tempFilePath)];
30+
2931
const sidebarModule = require(tempFilePath);
32+
3033
return sidebarModule.default;
3134
} catch (error) {
3235
console.error('Error converting sidebar config:', error);
@@ -38,33 +41,55 @@ function convertSidebarConfigToJson(filePath) {
3841
}
3942
}
4043

44+
const SLUG_TO_URL = {
45+
'architecture-overview': 'overview',
46+
'architecture-glossary': 'glossary',
47+
};
48+
4149
// Function to extract URLs from sidebar config
42-
function extractUrlsFromSidebar(sidebarConfig) {
50+
function extractUrlsFromSidebar(sidebarConfig, prefix) {
4351
const urls = [];
4452

4553
// Process each section (docs, api, components)
4654
Object.entries(sidebarConfig).forEach(([_, categories]) => {
4755
Object.entries(categories).forEach(([_, items]) => {
48-
processItemsForUrls(items, urls);
56+
processItemsForUrls(items, urls, prefix);
4957
});
5058
});
5159

60+
// Replace slugs with their mapped URLs
61+
urls.forEach((url, index) => {
62+
for (const [slug, mappedUrl] of Object.entries(SLUG_TO_URL)) {
63+
if (url.includes(slug)) {
64+
urls[index] = url.replace(slug, mappedUrl);
65+
break;
66+
}
67+
}
68+
});
69+
5270
return urls;
5371
}
5472

5573
// Recursive function to process items and extract URLs
56-
function processItemsForUrls(items, urls) {
57-
items.forEach(item => {
58-
if (typeof item === 'string') {
59-
urls.push(`${URL_PREFIX}${item}`);
60-
} else if (typeof item === 'object') {
61-
if (item.type === 'doc' && item.id) {
62-
urls.push(`${URL_PREFIX}${item.id}`);
63-
} else if (item.type === 'category' && Array.isArray(item.items)) {
64-
processItemsForUrls(item.items, urls);
74+
function processItemsForUrls(items, urls, prefix) {
75+
if (typeof items === 'object' && Array.isArray(items.items)) {
76+
processItemsForUrls(items.items, urls, prefix);
77+
return;
78+
}
79+
80+
if (Array.isArray(items)) {
81+
items.forEach(item => {
82+
if (typeof item === 'string') {
83+
urls.push(`${URL_PREFIX}${prefix}/${item}`);
84+
} else if (typeof item === 'object') {
85+
if (item.type === 'doc' && item.id) {
86+
urls.push(`${URL_PREFIX}${prefix}/${item.id}`);
87+
} else if (item.type === 'category' && Array.isArray(item.items)) {
88+
processItemsForUrls(item.items, urls, prefix);
89+
}
6590
}
66-
}
67-
});
91+
});
92+
}
6893
}
6994

7095
// Function to check URL status
@@ -143,34 +168,49 @@ async function processUrls(urls) {
143168
}
144169

145170
// Function to extract title from markdown frontmatter
146-
function extractTitleFromMarkdown(filePath) {
171+
function extractMetadataFromMarkdown(filePath) {
147172
try {
148173
const content = fs.readFileSync(filePath, 'utf8');
149174
const frontmatterMatch = content.match(/---\n([\s\S]*?)\n---/);
150175
if (frontmatterMatch) {
151176
const frontmatter = frontmatterMatch[1];
152177
const titleMatch = frontmatter.match(/title:\s*(.*)/);
153-
if (titleMatch) {
154-
return titleMatch[1].trim();
155-
}
178+
const slugMatch = frontmatter.match(/slug:\s*(.*)/);
179+
180+
return {
181+
title: titleMatch
182+
? titleMatch[1].trim()
183+
: filePath.split('/').pop().replace('.md', ''),
184+
slug: slugMatch ? slugMatch[1].trim().replace(/^\//, '') : null,
185+
};
156186
}
157-
// If no title found, use the filename
158-
return filePath.split('/').pop().replace('.md', '');
187+
// If no frontmatter found, use the filename
188+
return {
189+
title: filePath.split('/').pop().replace('.md', ''),
190+
slug: null,
191+
};
159192
} catch (error) {
160193
console.error(`Error reading file ${filePath}:`, error);
161-
return filePath.split('/').pop().replace('.md', '');
194+
return {
195+
title: filePath.split('/').pop().replace('.md', ''),
196+
slug: null,
197+
};
162198
}
163199
}
164200

165201
// Function to map special cases for file names that don't match the sidebar
166-
function mapDocPath(item) {
202+
function mapDocPath(item, prefix) {
167203
const specialCases = {
168204
'environment-setup': 'getting-started.md',
169205
'native-platform': 'native-platforms.md',
170206
'turbo-native-modules-introduction': 'turbo-native-modules.md',
171207
'fabric-native-components-introduction': 'fabric-native-components.md',
172208
};
173209

210+
if (prefix === '/contributing') {
211+
specialCases['overview'] = 'contributing-overview.md';
212+
}
213+
174214
if (typeof item === 'string') {
175215
return specialCases[item] || `${item}.md`;
176216
} else if (item.type === 'doc' && item.id) {
@@ -179,45 +219,56 @@ function mapDocPath(item) {
179219
return `${item}.md`;
180220
}
181221

182-
// Function to generate markdown documentation
183-
function generateMarkdown(sidebarConfig) {
184-
let markdown = `# ${TITLE}\n\n`;
185-
markdown += `> ${DESCRIPTION}\n\n`;
186-
markdown += `This documentation covers all aspects of using React Native, from installation to advanced usage.\n\n`;
222+
// Function to generate output for each sidebar
223+
function generateMarkdown(sidebarConfig, docPath, prefix) {
224+
let markdown = '';
187225

188226
// Process each section (docs, api, components)
189227
Object.entries(sidebarConfig).forEach(([section, categories]) => {
190228
markdown += `## ${section.charAt(0).toUpperCase() + section.slice(1)}\n\n`;
191229

192230
// Process each category within the section
193231
Object.entries(categories).forEach(([categoryName, items]) => {
194-
markdown += `### ${categoryName}\n\n`;
232+
markdown += `### ${categoryName === '0' ? 'General' : categoryName}\n\n`;
233+
234+
if (typeof items === 'object' && Array.isArray(items.items)) {
235+
items = items.items;
236+
}
237+
const reorderedArray = items.every(item => typeof item === 'string')
238+
? items
239+
: [...items].sort((a, b) =>
240+
typeof a === 'string' && typeof b !== 'string'
241+
? -1
242+
: typeof a !== 'string' && typeof b === 'string'
243+
? 1
244+
: 0
245+
);
195246

196247
// Process each item in the category
197-
items.forEach(item => {
248+
reorderedArray.forEach(item => {
198249
if (typeof item === 'string') {
199250
// This is a direct page reference
200-
const docPath = `${DOCS_PATH}${mapDocPath(item)}`;
201-
const title = extractTitleFromMarkdown(docPath);
202-
markdown += `- [${title}](${URL_PREFIX}${item})\n`;
251+
const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`;
252+
const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
253+
markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item})\n`;
203254
} else if (typeof item === 'object') {
204255
if (item.type === 'doc' && item.id) {
205256
// This is a doc reference with an explicit ID
206-
const docPath = `${DOCS_PATH}${mapDocPath(item)}`;
207-
const title = extractTitleFromMarkdown(docPath);
208-
markdown += `- [${title}](${URL_PREFIX}${item.id})\n`;
257+
const fullDocPath = `${docPath}${mapDocPath(item, prefix)}`;
258+
const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
259+
markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? item.id})\n`;
209260
} else if (item.type === 'category' && Array.isArray(item.items)) {
210261
// This is a category with nested items
211262
markdown += `#### ${item.label}\n\n`;
212263
item.items.forEach(nestedItem => {
213264
if (typeof nestedItem === 'string') {
214-
const docPath = `${DOCS_PATH}${mapDocPath(nestedItem)}`;
215-
const title = extractTitleFromMarkdown(docPath);
216-
markdown += `- [${title}](${URL_PREFIX}${nestedItem})\n`;
265+
const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`;
266+
const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
267+
markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem})\n`;
217268
} else if (nestedItem.type === 'doc' && nestedItem.id) {
218-
const docPath = `${DOCS_PATH}${mapDocPath(nestedItem)}`;
219-
const title = extractTitleFromMarkdown(docPath);
220-
markdown += `- [${title}](${URL_PREFIX}${nestedItem.id})\n`;
269+
const fullDocPath = `${docPath}${mapDocPath(nestedItem, prefix)}`;
270+
const {title, slug} = extractMetadataFromMarkdown(fullDocPath);
271+
markdown += `- [${title}](${URL_PREFIX}${prefix}/${slug ?? nestedItem.id})\n`;
221272
}
222273
});
223274
}
@@ -230,33 +281,94 @@ function generateMarkdown(sidebarConfig) {
230281
return markdown.replace(/(#+ .*)\n/g, '\n$1\n').replace(/\n(\n)+/g, '\n\n');
231282
}
232283

233-
const inputFilePath = './sidebars.ts';
234-
const outputFilePath = inputFilePath.replace(/\.tsx?$/, '-urls.txt');
235-
236-
const sidebarConfig = convertSidebarConfigToJson(inputFilePath);
237-
if (sidebarConfig) {
238-
const urls = extractUrlsFromSidebar(sidebarConfig);
239-
240-
// First check URLs for 404 errors
241-
processUrls(urls)
242-
.then(result => {
243-
if (result.unavailableUrls.length === 0) {
244-
// Only generate documentation if all URLs are valid
245-
const markdown = generateMarkdown(sidebarConfig);
246-
fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), markdown);
247-
console.log(
248-
`Successfully generated documentation to: ${OUTPUT_FILENAME}`
249-
);
250-
} else {
251-
console.error('Documentation generation skipped due to broken links');
252-
process.exit(1);
253-
}
284+
const inputFilePaths = [
285+
{
286+
name: 'sidebars.ts',
287+
docPath: '../docs/',
288+
prefix: '/docs',
289+
},
290+
{
291+
name: 'sidebarsArchitecture.ts',
292+
docPath: './architecture/',
293+
prefix: '/architecture',
294+
},
295+
{
296+
name: 'sidebarsCommunity.ts',
297+
docPath: './community/',
298+
prefix: '/community',
299+
},
300+
{
301+
name: 'sidebarsContributing.ts',
302+
docPath: './contributing/',
303+
prefix: '/contributing',
304+
},
305+
];
306+
307+
let output = `# ${TITLE}\n\n`;
308+
output += `> ${DESCRIPTION}\n\n`;
309+
output += `This documentation covers all aspects of using React Native, from installation to advanced usage.\n\n`;
310+
311+
const generateOutput = () => {
312+
const results = [];
313+
const promises = [];
314+
315+
for (const {name, docPath, prefix} of inputFilePaths) {
316+
const inputFilePath = `./${name}`;
317+
const outputFilePath = inputFilePath.replace(/\.tsx?$/, '-urls.txt');
318+
319+
const sidebarConfig = convertSidebarConfigToJson(inputFilePath);
320+
if (sidebarConfig) {
321+
const urls = extractUrlsFromSidebar(sidebarConfig, prefix);
322+
323+
// First check URLs for 404 errors
324+
const promise = processUrls(urls)
325+
.then(result => {
326+
if (result.unavailableUrls.length === 0) {
327+
// Only generate documentation if all URLs are valid
328+
const markdown = generateMarkdown(sidebarConfig, docPath, prefix);
329+
results.push({ markdown, prefix });
330+
console.log(`Successfully generated output from ${inputFilePath}`);
331+
} else {
332+
console.error(
333+
'Documentation generation skipped due to broken links'
334+
);
335+
process.exit(1);
336+
}
337+
})
338+
.catch(err => {
339+
console.error('Error processing URLs:', err);
340+
process.exit(1);
341+
});
342+
343+
promises.push(promise);
344+
} else {
345+
console.error('Failed to convert sidebar config to JSON');
346+
process.exit(1);
347+
}
348+
}
349+
350+
// Wait for all promises to complete before writing the file
351+
Promise.all(promises)
352+
.then(() => {
353+
// Sort results to ensure docs section is first
354+
results.sort((a, b) => {
355+
if (a.prefix === '/docs') return -1;
356+
if (b.prefix === '/docs') return 1;
357+
return 0;
358+
});
359+
360+
// Combine all markdown content in the correct order
361+
output += results.map(r => r.markdown).join('\n');
362+
363+
fs.writeFileSync(path.join('build/', OUTPUT_FILENAME), output);
364+
console.log(
365+
`Successfully generated documentation to: ${OUTPUT_FILENAME}`
366+
);
254367
})
255368
.catch(err => {
256-
console.error('Error processing URLs:', err);
369+
console.error('Error during processing:', err);
257370
process.exit(1);
258371
});
259-
} else {
260-
console.error('Failed to convert sidebar config to JSON');
261-
process.exit(1);
262-
}
372+
};
373+
374+
generateOutput();

0 commit comments

Comments
 (0)