Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,14 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
Comment thread
coderabbitai[bot] marked this conversation as resolved.
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: 22
cache: npm
- run: npm ci
- run: npm run check
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.14
- run: bun install --frozen-lockfile
- run: bun run check
25 changes: 25 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

69 changes: 0 additions & 69 deletions package-lock.json

This file was deleted.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
"version": "0.1.0",
"private": true,
"type": "module",
"packageManager": "bun@1.3.14",
"scripts": {
"process": "node scripts/process-links.mjs",
"research": "node scripts/process-links.mjs",
"doctor": "node scripts/doctor.mjs",
"test": "node --test",
"check": "npm test && node --check scripts/web/server.mjs && node --check docs/js/app.mjs && node --check web/js/app.mjs && bash -n scripts/deploy/oracle-provision.sh"
"check": "bun run test && node --check scripts/web/server.mjs && node --check docs/js/app.mjs && node --check web/js/app.mjs && bash -n scripts/deploy/oracle-provision.sh"
},
"dependencies": {
"dotenv": "16.4.7",
Expand Down
33 changes: 24 additions & 9 deletions scripts/research/literature-search.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -370,12 +370,27 @@ const deduplicateResults = (results) => {

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

const decodeHtmlEntities = (text) =>
text
.replace(/&/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&apos;/g, "'")
.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d)));
const NAMED_HTML_ENTITIES = new Map([
['amp', '&'],
['lt', '<'],
['gt', '>'],
['quot', '"'],
['apos', "'"],
]);

export const decodeHtmlEntities = (text) =>
text.replace(/&(amp|lt|gt|quot|apos|#(?:\d+|[xX][0-9a-fA-F]+));/g, (entity, name) => {
if (name.startsWith('#')) {
const isHex = name[1] === 'x' || name[1] === 'X';
const codePoint = isHex
? Number.parseInt(name.slice(2), 16)
: Number(name.slice(1));
return Number.isSafeInteger(codePoint) &&
codePoint >= 0 &&
codePoint <= 0x10ffff &&
(codePoint < 0xd800 || codePoint > 0xdfff)
? String.fromCodePoint(codePoint)
: entity;
}
return NAMED_HTML_ENTITIES.get(name) ?? entity;
});
13 changes: 13 additions & 0 deletions tests/html-entities.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import assert from 'node:assert/strict';
import test from 'node:test';

import {decodeHtmlEntities} from '../scripts/research/literature-search.mjs';

test('decodes decimal and hexadecimal entities in one pass', () => {
assert.equal(decodeHtmlEntities('&#60; &#x3C; &#X1F642;'), '< < 🙂');
assert.equal(decodeHtmlEntities('&amp;lt;'), '&lt;');
});

test('preserves invalid Unicode references', () => {
assert.equal(decodeHtmlEntities('&#xD800; &#1114112;'), '&#xD800; &#1114112;');
});
Loading