diff --git a/README.md b/README.md index 55014c1..ef73502 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ const scripts = getScripts('

{ - if (value === null) return true +const { Parser } = require('htmlparser2') +const Tokenizer = require(join( + dirname(require.resolve('htmlparser2')), + 'Tokenizer.js' +)).default + +// htmlparser2 treats title/textarea/style/script as raw text everywhere. +// In svg/math that is only true for HTML, so parse those tags as markup. +const IN_TAG_NAME = 3 + +class ForeignTokenizer extends Tokenizer { + stateBeforeSpecialS (char) { + if (this.cbs.foreignContext[0] === true) { + this.state = IN_TAG_NAME + this.stateInTagName(char) + return + } + super.stateBeforeSpecialS(char) + } + + stateBeforeSpecialT (char) { + if (this.cbs.foreignContext[0] === true) { + this.state = IN_TAG_NAME + this.stateInTagName(char) + return + } + super.stateBeforeSpecialT(char) + } +} + +// svg/math are foreign content. These elements switch back to HTML. +// annotation-xml does that only with an HTML encoding, so it is handled apart. +const FOREIGN_ELEMENTS = new Set(['math', 'svg']) +const HTML_INTEGRATION_ELEMENTS = new Set([ + 'desc', + 'foreignobject', + 'mi', + 'mn', + 'mo', + 'ms', + 'mtext', + 'title' +]) +// style, textarea, xmp, and script are raw in HTML via the tokenizer. +const RAW_TEXT_ELEMENTS = new Set([ + 'iframe', + 'noembed', + 'noframes', + 'noscript', + 'plaintext' +]) +const getValue = (value, quote) => { + if (quote === undefined) return true if (value === '1') return true return value } -const fromPairs = collection => - collection.reduce((acc, { key, value }) => { - acc[key] = getValue(value) - return acc - }, {}) +const isHtmlEncoding = value => { + const mime = value.toLowerCase() + return mime === 'text/html' || mime === 'application/xhtml+xml' +} + +const isWhitespace = code => + code === 9 || code === 10 || code === 12 || code === 13 || code === 32 -const collectScripts = (nodes, acc = []) => { - for (const node of nodes) { - if (node.tagName === 'script') acc.push(node) - if (node.children) collectScripts(node.children, acc) +// Index of the ` { + for ( + let index = source.indexOf('<', from); + index !== -1; + index = source.indexOf('<', index + 1) + ) { + if (source.slice(index, index + 8).toLowerCase() !== ' + FOREIGN_ELEMENTS.has(name) || HTML_INTEGRATION_ELEMENTS.has(name) + module.exports = html => { - const scripts = collectScripts(parse(html)) - return scripts.map(({ attributes }) => fromPairs(attributes)) + const source = String(html) + const scripts = [] + let attributes = null + let templateDepth = 0 + let isInScript = false + // Taken when the script opens. htmlparser2 pops an integration point + // before the implied close, so the close event is too late to read. + let scriptInForeign = false + // htmlparser2 drops script-data mode on ``. Events before this index are still that script's text. + let scriptEnd = -1 + let rawText = null + let sawAnnotation = false + let annotationHtml = false + let sawAnnotationEncoding = false + const annotationHtmlStack = [] + const foreignContext = [false] + const leaveScriptIfEnded = () => { + if (scriptEnd !== -1 && parser.endIndex >= scriptEnd) { + isInScript = false + scriptEnd = -1 + } + } + const parser = new Parser( + { + onopentagname: name => { + leaveScriptIfEnded() + attributes = null + if (isInScript || rawText) return + if (FOREIGN_ELEMENTS.has(name)) foreignContext.unshift(true) + else if (HTML_INTEGRATION_ELEMENTS.has(name)) { + foreignContext.unshift(false) + } + if (name === 'annotation-xml') sawAnnotation = true + // These elements are raw text in HTML only. In svg/math they are markup. + if (RAW_TEXT_ELEMENTS.has(name) && foreignContext[0] !== true) { + rawText = name + return + } + if (name === 'template') templateDepth++ + if (name !== 'script') return + scriptInForeign = foreignContext[0] === true + // A foreign script does not swallow the markup inside it. + if (!scriptInForeign) isInScript = true + if (templateDepth === 0) { + attributes = {} + scripts.push(attributes) + } + }, + onattribute: (name, value, quote) => { + if (sawAnnotation && name === 'encoding' && !sawAnnotationEncoding) { + sawAnnotationEncoding = true + annotationHtml = isHtmlEncoding(value) + } + if (attributes) attributes[name] = getValue(value, quote) + }, + onopentag: name => { + if (name === 'annotation-xml' && sawAnnotation) { + annotationHtmlStack.unshift(annotationHtml) + if (annotationHtml) foreignContext.unshift(false) + sawAnnotation = false + annotationHtml = false + sawAnnotationEncoding = false + } + if (name !== 'script' || !isInScript) return + scriptEnd = findScriptEnd(source, parser.endIndex + 1) + }, + onclosetag: (name, isImplied) => { + leaveScriptIfEnded() + if (rawText) { + if (name === rawText && name !== 'plaintext') rawText = null + return + } + if (name === 'script') { + if (!isImplied || scriptInForeign) { + isInScript = false + scriptEnd = -1 + } + return + } + if (isInScript) return + if (name === 'annotation-xml') { + if (annotationHtmlStack.shift()) foreignContext.shift() + return + } + if (isForeignContextElement(name)) foreignContext.shift() + if (name === 'template' && templateDepth > 0) templateDepth-- + } + }, + { Tokenizer: ForeignTokenizer } + ) + parser.end(source) + return scripts } diff --git a/test/index.js b/test/index.js index 09a28e9..644862a 100644 --- a/test/index.js +++ b/test/index.js @@ -31,3 +31,317 @@ test('detect scripts from instagram iframe', t => { const scripts = parseIframe(html) t.snapshot(scripts) }) + +test('lowercase attribute names', t => { + t.deepEqual(parseIframe(''), [ + { src: '/a.js', async: true } + ]) +}) + +test('decode entities in attribute values', t => { + t.deepEqual( + parseIframe(''), + [{ src: '/a.js?x=1&y=<2>' }] + ) +}) + +test('map boolean and "1" attribute values to true', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ async: true, defer: true, 'data-empty': '', 'data-zero': '0' }] + ) +}) + +test('keep the last value of a repeated attribute', t => { + t.deepEqual(parseIframe(''), [ + { src: '/b.js' } + ]) +}) + +test('keep unquoted attribute values containing slashes', t => { + t.deepEqual( + parseIframe( + '' + ), + [ + { src: 'https://cdn.example.com/a.js', async: true }, + { type: 'module', src: '/m.js' } + ] + ) +}) + +test('detect scripts after a stray closing script tag', t => { + t.deepEqual( + parseIframe(''), + [{}, { src: '/b.js' }] + ) +}) + +test('ignore markup inside script content and comments', t => { + t.deepEqual( + parseIframe( + '' + ), + [{}, { src: '/d.js' }] + ) +}) + +test('ignore scripts inside template', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/after.js' }] + ) +}) + +test('a self-closing script keeps its content as text until ', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) +}) + +test('a script inside an svg title is a script', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }] + ) +}) + +test('a script inside an html title is text', t => { + t.deepEqual( + parseIframe( + '<script src="/a.js"></script>' + ), + [{ src: '/b.js' }] + ) +}) + +test('a script inside an svg script is still a script', t => { + t.deepEqual( + parseIframe( + '' + ), + [{}, { src: '/inner.js' }] + ) +}) + +test('a self-closing script inside svg ends right away', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ href: '/x.js' }, { src: '/y.js' }] + ) +}) + +test('a self-closing script inside math ends right away', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ href: '/x.js' }, { src: '/y.js' }] + ) +}) + +test('closing an integration point returns to foreign content', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ href: '/x.js' }, { src: '/y.js' }] + ) + t.deepEqual( + parseIframe( + '' + ), + [{ href: '/x.js' }, { src: '/y.js' }] + ) +}) + +test('a self-closing script inside an html integration point stays open', t => { + const points = [ + ['svg', 'foreignObject'], + ['svg', 'desc'], + ['math', 'mi'], + ['math', 'mo'], + ['math', 'mn'], + ['math', 'ms'], + ['math', 'mtext'] + ] + for (const [outer, inner] of points) { + t.deepEqual( + parseIframe( + `<${outer}><${inner}>` + ), + [{ src: '/a.js' }], + inner + ) + } +}) + +test('a script end tag still closes a self-closing html script', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { href: '/b.js' }] + ) +}) + +test('annotation-xml stays in math without an html encoding', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/inner.js' }] + ) + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) +}) + +test('closing an html annotation-xml returns to math', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) +}) + +test('annotation-xml with an html encoding keeps the script open', t => { + for (const encoding of ['text/html', 'application/xhtml+xml', 'TEXT/HTML']) { + t.deepEqual( + parseIframe( + `` + ), + [{ src: '/a.js' }], + encoding + ) + } +}) + +test('a script end tag may have whitespace before >', t => { + for (const gap of [' ', '\n', '\t', '\r', '\f']) { + t.deepEqual( + parseIframe( + `
` + ), + [{ src: '/a.js' }, { src: '/b.js' }], + JSON.stringify(gap) + ) + } +}) + +test('raw text elements inside svg are markup', t => { + t.deepEqual( + parseIframe( + '' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) +}) + +test('plaintext has no end tag', t => { + t.deepEqual( + parseIframe('</plaintext><script src="/b.js"></script>'), + [] + ) +}) + +test('an unclosed script runs to the end of the document', t => { + t.deepEqual(parseIframe('<script src="/a.js"/><div><script src="/b.js">'), [ + { src: '/a.js' } + ]) +}) + +test('an unquoted slash is part of the attribute value', t => { + t.deepEqual(parseIframe('<script src=a/><script src="/b.js"></script>'), [ + { src: 'a/' } + ]) +}) + +test('an end tag inside a script does not close the element around it', t => { + t.deepEqual( + parseIframe( + '<template><script src="/a.js"/></template></script><script src="/b.js"></script>' + ), + [] + ) +}) + +test('raw text elements hide the scripts inside them', t => { + for (const tag of ['iframe', 'noembed', 'noframes', 'noscript']) { + t.deepEqual( + parseIframe( + `<${tag}><script src="/a.js"></script></${tag}><script src="/b.js"></script>` + ), + [{ src: '/b.js' }], + tag + ) + } + t.deepEqual( + parseIframe( + '<script src="/a.js"></script><plaintext><script src="/b.js"></script>' + ), + [{ src: '/a.js' }] + ) +}) + +test('an implied end tag does not end an html script', t => { + t.deepEqual( + parseIframe('<script src="/a.js"/><body><script src="/b.js"></script>'), + [{ src: '/a.js' }] + ) + t.deepEqual( + parseIframe( + '<div><script src="/a.js"/></div><script src="/b.js"></script>' + ), + [{ src: '/a.js' }] + ) + t.deepEqual( + parseIframe( + '<head><script src="/a.js"></script></head><body><script src="/b.js"></script>' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) +}) + +test('non-string input is read as a string', t => { + t.deepEqual(parseIframe(Buffer.from('<script src="/buf.js"></script>')), [ + { src: '/buf.js' } + ]) + t.deepEqual(parseIframe(undefined), []) +}) + +test('a stray </template> after a self-closing script keeps later scripts', t => { + t.deepEqual( + parseIframe( + '<script src="/a.js"/><template><script></script></template><script src="/b.js"></script>' + ), + [{ src: '/a.js' }, { src: '/b.js' }] + ) +}) diff --git a/test/snapshots/index.js.md b/test/snapshots/index.js.md index 064df99..f49b5b9 100644 --- a/test/snapshots/index.js.md +++ b/test/snapshots/index.js.md @@ -25,7 +25,7 @@ Generated by [AVA](https://avajs.dev). async: true, crossorigin: 'anonymous', defer: true, - src: 'https://connect.facebook.net/en_US/sdk.js#xfbml=1&amp;version=v5.0', + src: 'https://connect.facebook.net/en_US/sdk.js#xfbml=1&version=v5.0', }, ] diff --git a/test/snapshots/index.js.snap b/test/snapshots/index.js.snap index f7325ff..e7c9a6a 100644 Binary files a/test/snapshots/index.js.snap and b/test/snapshots/index.js.snap differ