diff --git a/biome.json b/biome.json index d3c205091..ad7a2783b 100644 --- a/biome.json +++ b/biome.json @@ -11,6 +11,7 @@ "!drizzle", "!src/store/sqlite/migrations.generated.ts", "!src/search/snowball", + "!test/fixtures/posts", "!.worktrees", "!.claude", "!config/wirecat", diff --git a/changes/feat-html-markdown.md b/changes/feat-html-markdown.md new file mode 100644 index 000000000..db6821158 --- /dev/null +++ b/changes/feat-html-markdown.md @@ -0,0 +1,8 @@ +### Added + +- **`htmlToMarkdown` and `markdownToHtml` in `@wirecat/cli-core/documents`**, for the content category: a blog keeps + HTML (WordPress, Ghost), and a CLI shows and takes Markdown. WordPress block comments and Ghost card comments go, + and so do scripts, styles, frames, event attributes, raw HTML from Markdown, and links and images whose scheme is + not http(s), mailto or tel: a post body is untrusted. Tested on one post as WordPress 6.9 stores and renders it and + as Ghost 6.78 renders it; Markdown that went through HTML once comes back the same. The converters are unified's + `hast-util-from-html`, `hast-util-to-mdast`, `mdast-util-to-hast` and `hast-util-to-html`, loaded on first use. diff --git a/docs/dev/DOCUMENTS.md b/docs/dev/DOCUMENTS.md index 4801b5e45..4e2f91359 100644 --- a/docs/dev/DOCUMENTS.md +++ b/docs/dev/DOCUMENTS.md @@ -328,6 +328,14 @@ differ change (a tick is flipped inside its line), so the rest keeps its markers byte-order mark. The result is parsed again and must read as `next`; otherwise it returns undefined and the caller reports a conflict instead of writing. `blocksMarkdown` writes blocks as Markdown that reads back as the same blocks. +A service that keeps HTML, such as a blog (WordPress, Ghost), converts with `htmlToMarkdown` and `markdownToHtml` +(async; loaded on first use). Both treat the body as untrusted: comments (WordPress's block comments, Ghost's card +comments), scripts, styles, frames and event attributes go, Markdown's raw HTML is not written, and a link or image +whose scheme is not http(s), mailto or tel keeps only its text. Markdown that went through HTML once comes back the +same; the tests run on real WordPress and Ghost posts in `test/fixtures/posts/`. HTML that went through Markdown does not: +embeds, frames, classes and block comments are lost, so a CLI that reads a post, edits it and writes it back +flattens it. Such a CLI warns or refuses to write back a body that had them. + ## Copy a document collection to another database Use `previewDocumentTransfer` and `applyDocumentTransfer` from `@wirecat/cli-core/store` to retain diff --git a/package.json b/package.json index bddaa9634..42a54c46a 100644 --- a/package.json +++ b/package.json @@ -277,6 +277,9 @@ "disposable-email-domains-js": "1.26.0", "echarts": "6.1.0", "fflate": "0.8.3", + "hast-util-from-html": "2.1.0", + "hast-util-to-html": "9.0.6", + "hast-util-to-mdast": "10.1.3", "iconv-lite": "0.7.3", "image-size": "2.0.4", "linkedom": "0.18.14", @@ -285,6 +288,8 @@ "mammoth": "1.13.0", "mdast-util-from-markdown": "2.1.0", "mdast-util-gfm": "3.1.0", + "mdast-util-to-hast": "13.2.2", + "mdast-util-to-markdown": "2.2.0", "micromark-extension-gfm": "3.0.0", "mpg123-decoder": "1.0.3", "ogg-opus-decoder": "1.7.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa3fdfe11..2af298b0c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -73,6 +73,15 @@ importers: fflate: specifier: 0.8.3 version: 0.8.3 + hast-util-from-html: + specifier: 2.1.0 + version: 2.1.0 + hast-util-to-html: + specifier: 9.0.6 + version: 9.0.6 + hast-util-to-mdast: + specifier: 10.1.3 + version: 10.1.3 iconv-lite: specifier: 0.7.3 version: 0.7.3 @@ -97,6 +106,12 @@ importers: mdast-util-gfm: specifier: 3.1.0 version: 3.1.0 + mdast-util-to-hast: + specifier: 13.2.2 + version: 13.2.2 + mdast-util-to-markdown: + specifier: 2.2.0 + version: 2.2.0 micromark-extension-gfm: specifier: 3.0.0 version: 3.0.0 @@ -1178,6 +1193,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/katex@0.16.8': resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==} @@ -1325,6 +1343,9 @@ packages: cpu: [x64] os: [win32] + '@ungap/structured-clone@1.4.0': + resolution: {integrity: sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==} + '@valibot/to-json-schema@1.8.0': resolution: {integrity: sha512-a0M+uwCuQZEPAo65NYkFSJ14O5c213KoSZmPdoCfuFvUhfULi4T2Z6Tpjv6lTVMW9RoLm74RpRAjNEa43KcN+g==} peerDependencies: @@ -1505,6 +1526,9 @@ packages: resolution: {integrity: sha512-/Ce6KNm3vIbWdMlNna6RVIZ/ICQxnJxCicet5LBKK9ZffBkqzDw0xh9EiKSljdRtiIQ1S1z4YgcscUUGzNCWrA==} engines: {node: '>=22'} + character-entities-html4@2.1.0: + resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} + character-entities-legacy@3.0.0: resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} @@ -1524,6 +1548,12 @@ packages: codec-parser@2.5.0: resolution: {integrity: sha512-Ru9t80fV8B0ZiixQl8xhMTLru+dzuis/KQld32/x5T/+3LwZb0/YvQdSKytX9JqCnRdiupvAvyYJINKrXieziQ==} + collapse-white-space@2.1.0: + resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + + comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + commander@10.0.1: resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} engines: {node: '>=14'} @@ -1867,6 +1897,10 @@ packages: resolution: {integrity: sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==} engines: {node: '>=20.19.0'} + entities@9.0.0: + resolution: {integrity: sha512-oJFNDJg3kTPwAD1cNn2+WmUq4wFoo09npc3Ydf1D7ED4smRiWrsKnaLxykrTWgyGEcusc87P5dxlAleol9kpkg==} + engines: {node: '>=20.19.0'} + env-paths@4.0.0: resolution: {integrity: sha512-pxP8eL2SwwaTRi/KHYwLYXinDs7gL3jxFcBYmEdYfZmZXbaVDvdppd0XBU8qVz03rDfKZMXg1omHCbsJjZrMsw==} engines: {node: '>=20'} @@ -1973,9 +2007,54 @@ packages: resolution: {integrity: sha512-c8B/VNLmxRcmqqenRA9t+9IyOjf9+V6lTxPaUJLqOCONdQkWZ0ETYgX0qbtJqPsgCNusT9MZ5Jeidw8Eb9tn2g==} engines: {node: '>=20'} + hast-util-embedded@3.0.0: + resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} + + hast-util-from-html@2.1.0: + resolution: {integrity: sha512-873p9AuH93gBnATbH8rU3YvjTKxyrIeWd/M9tiLMzn3G82Os0xhQurctfnbYUSrDXoyjbJGpJ2qmTpefMw7ZaQ==} + + hast-util-from-parse5@8.0.4: + resolution: {integrity: sha512-Yq7XlXor0Cpb71sI2E0zpv8QxQ5a6rfhDEETk4bcgrFX2SvJAY2PHocWwFtL6U9qyfiyByW3wZ6sZEfP2kW81g==} + + hast-util-has-property@3.0.0: + resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} + + hast-util-is-body-ok-link@3.0.1: + resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} + + hast-util-is-element@3.0.0: + resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} + + hast-util-minify-whitespace@1.0.1: + resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} + + hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + + hast-util-phrasing@3.0.1: + resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} + + hast-util-to-html@9.0.6: + resolution: {integrity: sha512-1SZUxCN3TtJAE7e2BQN2MBHsoZ1gnmi9GoCP5Di59CSWptIezSAW09yDZyVbanb78V9eCwrKqdQsjtfVTxYHWA==} + + hast-util-to-mdast@10.1.3: + resolution: {integrity: sha512-8Qc3IBGvw+tvOCi606vYSvizX89hxW22GEuXnqSGq0m3cXotDaNo6LUEIuCjIL+kmrCaMeBTtu+aiJ71HoyE7Q==} + + hast-util-to-text@4.0.3: + resolution: {integrity: sha512-5yrlWOvGFs6CqHriUpSsNrjzT/Tdo+IOQudbznxOn8K5qblp18MJyzOg8IgDOKQ8MYgQcJA7SusuCjSpDr8mSA==} + + hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + + hastscript@9.0.3: + resolution: {integrity: sha512-1ALZXSmCGmW3h8AMz4R4+QKSKFi7sNgXLVwmwB8FYVtl1CwQ3WDkSAEY/d5tp1g/vbQgq2s1WlHgfO3jbw1TsA==} + html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} + html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + htmlparser2@10.1.0: resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==} @@ -2338,6 +2417,9 @@ packages: mdast-util-phrasing@4.1.0: resolution: {integrity: sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==} + mdast-util-to-hast@13.2.2: + resolution: {integrity: sha512-l2pnmOuqs+SGLVpTVXnRLQu5v1AzvDe181404fbXzzqObVxrAnnVxapAP27pUwzLBL/ABIkgUhGlPVU7QSvIFg==} + mdast-util-to-markdown@2.2.0: resolution: {integrity: sha512-2Jn/ADrHFFyyniBMi0bncTbJqU78K3/TEYvzvwrWWzD9MYPUa/Gm4r3nOeKFQg/PDM5BsbDYdc8ackW4NTY6NA==} @@ -2498,6 +2580,9 @@ packages: parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse5@8.1.1: + resolution: {integrity: sha512-EcR/6WDcU7b2N3vQSxKALCk9lXGtUMQxXio008OEMBE0kavu/7JLfBEeFPzi43nMqVDSbEn3QCK+BMCc9zO//w==} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -2585,6 +2670,9 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + property-information@7.2.0: + resolution: {integrity: sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==} + psl@1.15.0: resolution: {integrity: sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==} @@ -2609,6 +2697,9 @@ packages: readable-stream@2.3.8: resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + rehype-minify-whitespace@6.0.2: + resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} + require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} @@ -2676,6 +2767,9 @@ packages: resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} + space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -2705,6 +2799,9 @@ packages: string_decoder@1.1.1: resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + stringify-entities@4.0.4: + resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -2744,6 +2841,12 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + + trim-trailing-lines@2.1.0: + resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} + ts-algebra@2.0.0: resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} @@ -2781,6 +2884,9 @@ packages: unist-util-is@6.0.1: resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==} + unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + unist-util-stringify-position@4.0.0: resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} @@ -2819,6 +2925,15 @@ packages: typescript: optional: true + vfile-location@5.0.4: + resolution: {integrity: sha512-vEvxatYZCIpNvzGk2UD+u4I/sZkSKoOsdRCbkOIsN73QpJbriJ1qvvBAL1PeI0y55Leoqf7pORbaSPpG/smmOg==} + + vfile-message@4.0.4: + resolution: {integrity: sha512-66ycRswIkNmHH9PHku7ADtv5RVwgIrOcuv3J8Svz6h4qgGRrKVa+9/neuKyJZUXyOpYwLTvS+whZ4R2urHHgQg==} + + vfile@6.0.3: + resolution: {integrity: sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==} + vite@8.3.5: resolution: {integrity: sha512-z1YpjieJ9g0ebTmQpLBv/oq9E3F9EIlOe+7LahW8SrE3LTiG3udBRUXQOlYXP9b+IC2q2phHeVxj9xGlw34OEQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2909,6 +3024,9 @@ packages: vscode-uri@3.2.0: resolution: {integrity: sha512-m2gXo3bn0G1kT9InzMf07fTbqMbGtyckj3bH5ktLO+1Ssv+yiATZ4dhwaQv9UZWxJh6E9IFGnQyjgWVDWVBDrg==} + web-namespaces@2.0.1: + resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -3624,6 +3742,10 @@ snapshots: '@types/estree@1.0.9': {} + '@types/hast@3.0.5': + dependencies: + '@types/unist': 3.0.3 + '@types/katex@0.16.8': {} '@types/mdast@4.0.4': @@ -3714,6 +3836,8 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true + '@ungap/structured-clone@1.4.0': {} + '@valibot/to-json-schema@1.8.0(valibot@1.5.0(typescript@7.0.2))': dependencies: valibot: 1.5.0(typescript@7.0.2) @@ -3907,6 +4031,8 @@ snapshots: chalk@6.0.1: {} + character-entities-html4@2.1.0: {} + character-entities-legacy@3.0.0: {} character-entities@2.0.2: {} @@ -3923,6 +4049,10 @@ snapshots: codec-parser@2.5.0: {} + collapse-white-space@2.1.0: {} + + comma-separated-tokens@2.0.3: {} + commander@10.0.1: {} commander@12.1.0: {} @@ -4165,6 +4295,8 @@ snapshots: entities@8.1.0: {} + entities@9.0.0: {} + env-paths@4.0.0: dependencies: is-safe-filename: 0.1.1 @@ -4281,8 +4413,115 @@ snapshots: slash: 5.1.0 unicorn-magic: 0.4.1 + hast-util-embedded@3.0.0: + dependencies: + '@types/hast': 3.0.5 + hast-util-is-element: 3.0.0 + + hast-util-from-html@2.1.0: + dependencies: + '@types/hast': 3.0.5 + devlop: 1.1.0 + hast-util-from-parse5: 8.0.4 + parse5: 8.1.1 + vfile: 6.0.3 + vfile-message: 4.0.4 + + hast-util-from-parse5@8.0.4: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + devlop: 1.1.0 + hastscript: 9.0.3 + property-information: 7.2.0 + vfile: 6.0.3 + vfile-location: 5.0.4 + web-namespaces: 2.0.1 + + hast-util-has-property@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + hast-util-is-body-ok-link@3.0.1: + dependencies: + '@types/hast': 3.0.5 + + hast-util-is-element@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + hast-util-minify-whitespace@1.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-embedded: 3.0.0 + hast-util-is-element: 3.0.0 + hast-util-whitespace: 3.0.0 + unist-util-is: 6.0.1 + + hast-util-parse-selector@4.0.0: + dependencies: + '@types/hast': 3.0.5 + + hast-util-phrasing@3.0.1: + dependencies: + '@types/hast': 3.0.5 + hast-util-embedded: 3.0.0 + hast-util-has-property: 3.0.0 + hast-util-is-body-ok-link: 3.0.1 + hast-util-is-element: 3.0.0 + + hast-util-to-html@9.0.6: + dependencies: + '@types/hast': 3.0.5 + '@types/unist': 3.0.3 + ccount: 2.0.1 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 3.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.2 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + stringify-entities: 4.0.4 + zwitch: 2.0.4 + + hast-util-to-mdast@10.1.3: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.4.0 + collapse-white-space: 2.1.0 + hast-util-phrasing: 3.0.1 + hast-util-to-html: 9.0.6 + hast-util-to-text: 4.0.3 + hast-util-whitespace: 3.0.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-hast: 13.2.2 + mdast-util-to-string: 4.0.0 + rehype-minify-whitespace: 6.0.2 + trim-trailing-lines: 2.1.0 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + + hast-util-to-text@4.0.3: + dependencies: + '@types/hast': 3.0.5 + + hast-util-whitespace@3.0.0: + dependencies: + '@types/hast': 3.0.5 + + hastscript@9.0.3: + dependencies: + '@types/hast': 3.0.5 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 7.2.0 + space-separated-tokens: 2.0.2 + html-escaper@3.0.3: {} + html-void-elements@3.0.0: {} + htmlparser2@10.1.0: dependencies: domelementtype: 2.3.0 @@ -4671,6 +4910,18 @@ snapshots: '@types/mdast': 4.0.4 unist-util-is: 6.0.1 + mdast-util-to-hast@13.2.2: + dependencies: + '@types/hast': 3.0.5 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.4.0 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.1 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + mdast-util-to-markdown@2.2.0: dependencies: '@types/mdast': 4.0.4 @@ -4974,6 +5225,10 @@ snapshots: is-decimal: 2.0.1 is-hexadecimal: 2.0.1 + parse5@8.1.1: + dependencies: + entities: 9.0.0 + path-key@3.1.1: {} pend@1.2.0: {} @@ -5043,6 +5298,8 @@ snapshots: process-nextick-args@2.0.1: {} + property-information@7.2.0: {} + psl@1.15.0: dependencies: punycode: 2.3.1 @@ -5067,6 +5324,11 @@ snapshots: string_decoder: 1.1.1 util-deprecate: 1.0.2 + rehype-minify-whitespace@6.0.2: + dependencies: + '@types/hast': 3.0.5 + hast-util-minify-whitespace: 1.0.1 + require-from-string@2.0.2: {} resolve-from@5.0.0: {} @@ -5128,6 +5390,8 @@ snapshots: source-map-js@1.2.2: {} + space-separated-tokens@2.0.2: {} + split2@4.2.0: {} sql-escaper@1.5.2: {} @@ -5162,6 +5426,11 @@ snapshots: dependencies: safe-buffer: 5.1.2 + stringify-entities@4.0.4: + dependencies: + character-entities-html4: 2.1.0 + character-entities-legacy: 3.0.0 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -5194,6 +5463,10 @@ snapshots: dependencies: is-number: 7.0.0 + trim-lines@3.0.1: {} + + trim-trailing-lines@2.1.0: {} + ts-algebra@2.0.0: {} tslib@2.3.0: {} @@ -5242,6 +5515,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + unist-util-position@5.0.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-stringify-position@4.0.0: dependencies: '@types/unist': 3.0.3 @@ -5271,6 +5548,21 @@ snapshots: optionalDependencies: typescript: 7.0.2 + vfile-location@5.0.4: + dependencies: + '@types/unist': 3.0.3 + vfile: 6.0.3 + + vfile-message@4.0.4: + dependencies: + '@types/unist': 3.0.3 + unist-util-stringify-position: 4.0.0 + + vfile@6.0.3: + dependencies: + '@types/unist': 3.0.3 + vfile-message: 4.0.4 + vite@8.3.5(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 @@ -5312,6 +5604,8 @@ snapshots: vscode-uri@3.2.0: {} + web-namespaces@2.0.1: {} + which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/src/documents/html-markdown.test.ts b/src/documents/html-markdown.test.ts new file mode 100644 index 000000000..c922b816b --- /dev/null +++ b/src/documents/html-markdown.test.ts @@ -0,0 +1,64 @@ +import { readFileSync } from "node:fs" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { htmlToMarkdown, markdownToHtml } from "./html-markdown.js" + +// One post, as WordPress 6.9.4 (wordpress:6-php8.3-apache) stores it (`wp post get --field=post_content`, block markup +// written for it) and renders it (`the_content`), and as Ghost 6.78.0 renders it after `POST /posts/?source=html` of +// WordPress's rendering. Taken 2026-11-18 from throwaway local Docker instances. +const fixture = (name: string) => readFileSync(join(import.meta.dirname, "../../test/fixtures/posts", name), "utf8") +const POSTS = ["wordpress-raw.html", "wordpress-rendered.html", "ghost.html"] + +describe("htmlToMarkdown on real posts", () => { + it.each(POSTS)("%s keeps the headings, links, lists, image, code and table, and none of the markup", async (name) => { + const markdown = await htmlToMarkdown(fixture(name)) + expect(markdown).toContain("## Why we moved") + expect(markdown).toContain("[our notes](https://example.com/notes?a=1\\&b=2)") + expect(markdown).toMatch(/\* Second point with a \[relative link\]\([^)]*\/relative\)\n {2}\* Nested/) + expect(markdown).toContain("1. One\n2. Two") + expect(markdown).toContain("") + expect(markdown).toContain('```\ndocker compose up -d\necho "a < b && c"\n```') + expect(markdown).toMatch(/\| Host +\| CPU \|/) + expect(markdown).toContain("Ünïcödé — «кавычки» and an emoji 🚀.") + expect(markdown).not.toMatch(/
t
'
+ expect(await htmlToMarkdown(html)).toBe("e\n\nt\n\n\n")
+ })
+
+ it("leaves a reference to an unsafe definition unlinked, even when a later definition has the same label", async () => {
+ expect(await markdownToHtml("[a][r] [b][R]\n\n[r]: javascript:x\n[R]: https://ok.example\n")).toBe(
+ "[a][r] [b][R]
", + ) + }) + + it("writes no raw HTML from Markdown", async () => { + expect( + await markdownToHtml( + 'a b\n\n', + ), + ).toBe("a x() b
") + }) +}) diff --git a/src/documents/html-markdown.ts b/src/documents/html-markdown.ts new file mode 100644 index 000000000..21e08fc45 --- /dev/null +++ b/src/documents/html-markdown.ts @@ -0,0 +1,82 @@ +import type { Nodes, Parent, Root } from "mdast" + +// Loaded on first use, as markdown-blocks.ts loads its parser: most commands never convert a body. +let converters: ReturnTypeWe ran on three servers, with one database and our notes. A line
break, and inline code.

Move fast, but keep the backups.Ops lead
docker compose up -d
+echo "a < b && c"
+
+| Host | CPU |
|---|---|
| web-1 | 4 |
| db-1 | 8 |
Custom HTML block
Ünïcödé — «кавычки» and an emoji 🚀.
\ No newline at end of file diff --git a/test/fixtures/posts/wordpress-raw.html b/test/fixtures/posts/wordpress-raw.html new file mode 100644 index 000000000..499dc88b7 --- /dev/null +++ b/test/fixtures/posts/wordpress-raw.html @@ -0,0 +1,63 @@ + +We ran on three servers, with one database and our notes. A line
break, and inline code.

++ + + +Move fast, but keep the backups.
+Ops lead
docker compose up -d
+echo "a < b && c"
+
+
+
+| Host | CPU |
|---|---|
| web-1 | 4 |
| db-1 | 8 |
Custom HTML block
+ + + +Ünïcödé — «кавычки» and an emoji 🚀.
+ + diff --git a/test/fixtures/posts/wordpress-rendered.html b/test/fixtures/posts/wordpress-rendered.html new file mode 100644 index 000000000..3014551a4 --- /dev/null +++ b/test/fixtures/posts/wordpress-rendered.html @@ -0,0 +1,62 @@ + +We ran on three servers, with one database and our notes. A line
break, and inline code.

++ + + +Move fast, but keep the backups.
+Ops lead
docker compose up -d
+echo "a < b && c"
+
+
+
+| Host | CPU |
|---|---|
| web-1 | 4 |
| db-1 | 8 |
Custom HTML block
+ + + +Ünïcödé — «кавычки» and an emoji 🚀.
+