selector.js

/*
 * selector-picker — selector generation core.
 *
 * This module is deliberately free of any browser-extension or DOM-construction
 * code so it can run unchanged in two places:
 *
 *   1. inside the content script (attached to `window.SelectorPicker`), and
 *   2. inside Node + jsdom for the unit tests (via `require`).
 *
 * The only thing it needs is a standard DOM Element with an owner document that
 * exposes `querySelectorAll` / `matches`. XPath verification is optional and can
 * be supplied through the `evaluate` option (the browser uses `document.evaluate`,
 * the tests inject the `xpath` package).
 */
(function (factory) {
  'use strict';
  var api = factory();
  if (typeof module !== 'undefined' && module.exports) {
    module.exports = api;
  }
  if (typeof window !== 'undefined') {
    window.SelectorPicker = api;
  }
})(function () {
  'use strict';

  // Attributes that tend to be stable across page loads and deploys. Ordered by
  // how trustworthy they usually are for scraping.
  var STABLE_ATTRS = [
    'data-testid',
    'data-test',
    'data-qa',
    'data-cy',
    'data-automation-id',
    'data-automationid',
    'data-test-id',
    'data-role',
    'name',
    'aria-label',
    'itemprop',
    'for'
  ];

  var MAX_CLASSES = 3;      // never stack more than this many classes in one part
  var MAX_ATTR_VALUE = 100; // ignore very long (often generated) attribute values
  var MAX_XPATH_TEXT = 40;  // only key XPath on short, leaf text

  // ---------------------------------------------------------------------------
  // Small helpers
  // ---------------------------------------------------------------------------

  function escapeCss(value) {
    if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
      return CSS.escape(value);
    }
    // Minimal fallback: escape everything that is not a safe identifier char.
    return String(value).replace(/[^a-zA-Z0-9_-]/g, function (ch) {
      return '\\' + ch;
    });
  }

  function cssAttrEscape(value) {
    return String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
  }

  // A name we are happy to drop into a selector as `#id` or `.class` unescaped.
  function isUsableName(name) {
    return typeof name === 'string' && /^[A-Za-z_][\w-]*$/.test(name);
  }

  // Heuristic: does this class/id/attribute value look machine-generated and
  // therefore unlikely to survive the next deploy? We avoid those when we can.
  function looksDynamic(token) {
    if (!token) {
      return true;
    }
    if (/\d{4,}/.test(token)) {
      // long digit runs, e.g. "item-83920", "ember1043"
      return true;
    }
    if (/(^|[-_])(css|sc|jsx|emotion|makeStyles|styles?)[-_]?[a-z0-9]{4,}/i.test(token)) {
      // css-in-js output: "css-1q2w3e", "sc-bdVaJa", "jsx-2043"
      return true;
    }
    if (/^[a-z0-9]{7,}$/i.test(token) && /\d/.test(token) && /[a-z]/i.test(token)) {
      // opaque hash blob: "a1b2c3d", "x8f0k2q"
      return true;
    }
    return false;
  }

  function safeMatches(el, selector) {
    try {
      return el.matches(selector);
    } catch (e) {
      return false;
    }
  }

  function isUnique(selector, el, doc) {
    try {
      var found = doc.querySelectorAll(selector);
      return found.length === 1 && found[0] === el;
    } catch (e) {
      return false;
    }
  }

  function stableClasses(el) {
    var list = el.classList ? Array.prototype.slice.call(el.classList) : [];
    return list.filter(function (c) {
      return isUsableName(c) && !looksDynamic(c);
    });
  }

  function stableAttrPart(el) {
    if (!el.hasAttribute) {
      return null;
    }
    for (var i = 0; i < STABLE_ATTRS.length; i++) {
      var attr = STABLE_ATTRS[i];
      if (el.hasAttribute(attr)) {
        var val = el.getAttribute(attr);
        if (val && val.length <= MAX_ATTR_VALUE && !looksDynamic(val)) {
          return '[' + attr + '="' + cssAttrEscape(val) + '"]';
        }
      }
    }
    return null;
  }

  function nthOfTypeIndex(el) {
    var index = 1;
    var sib = el.previousElementSibling;
    while (sib) {
      if (sib.tagName === el.tagName) {
        index++;
      }
      sib = sib.previousElementSibling;
    }
    return index;
  }

  function nthChildIndex(el) {
    var index = 1;
    var sib = el.previousElementSibling;
    while (sib) {
      index++;
      sib = sib.previousElementSibling;
    }
    return index;
  }

  // ---------------------------------------------------------------------------
  // CSS selector generation
  // ---------------------------------------------------------------------------

  // Best simple selector fragment for a single node, guaranteed to select it
  // among its parent's children. `anchored` means the fragment is unique in the
  // whole document (a good stopping point when walking up the tree).
  function partForNode(el, doc) {
    var tag = el.tagName.toLowerCase();

    if (el.id && isUsableName(el.id) && !looksDynamic(el.id)) {
      var idSel = '#' + escapeCss(el.id);
      if (isUnique(idSel, el, doc)) {
        return { part: idSel, anchored: true };
      }
    }

    var base = tag;
    var attrPart = stableAttrPart(el);
    var classes = stableClasses(el);
    if (attrPart) {
      base = tag + attrPart;
    } else if (classes.length) {
      base = tag + '.' + classes.slice(0, MAX_CLASSES).map(escapeCss).join('.');
    }

    var parent = el.parentElement;
    if (!parent) {
      return { part: base, anchored: false };
    }

    var siblings = Array.prototype.slice.call(parent.children);
    var baseMatches = siblings.filter(function (c) {
      return safeMatches(c, base);
    });
    if (baseMatches.length === 1) {
      return { part: base, anchored: false };
    }

    // Prefer :nth-of-type — it is more resilient to inserted/removed siblings of
    // other tags than :nth-child.
    var withNthType = base + ':nth-of-type(' + nthOfTypeIndex(el) + ')';
    var typeMatches = siblings.filter(function (c) {
      return safeMatches(c, withNthType);
    });
    if (typeMatches.length === 1) {
      return { part: withNthType, anchored: false };
    }

    // Last resort: positional nth-child always disambiguates a single child.
    return { part: base + ':nth-child(' + nthChildIndex(el) + ')', anchored: false };
  }

  function getCssSelector(el) {
    if (!el || el.nodeType !== 1) {
      return null;
    }
    var doc = el.ownerDocument || (typeof document !== 'undefined' ? document : null);
    if (!doc) {
      return null;
    }

    var tag = el.tagName.toLowerCase();
    if (tag === 'html' || tag === 'body') {
      return tag;
    }

    // 1. A unique id beats everything.
    if (el.id && isUsableName(el.id) && !looksDynamic(el.id)) {
      var idSel = '#' + escapeCss(el.id);
      if (isUnique(idSel, el, doc)) {
        return idSel;
      }
    }

    // 2. A stable attribute that is unique on its own.
    var attrPart = stableAttrPart(el);
    if (attrPart) {
      var tagAttr = tag + attrPart;
      if (isUnique(tagAttr, el, doc)) {
        return tagAttr;
      }
      if (isUnique(attrPart, el, doc)) {
        return attrPart;
      }
    }

    // 3. A class combination that is unique on its own.
    var classes = stableClasses(el);
    if (classes.length) {
      var classSel = tag + '.' + classes.slice(0, MAX_CLASSES).map(escapeCss).join('.');
      if (isUnique(classSel, el, doc)) {
        return classSel;
      }
    }

    // 4. Build the shortest descendant path that is unique, stopping as soon as
    //    it resolves to exactly one element.
    var path = [];
    var node = el;
    while (node && node.nodeType === 1 && node.tagName.toLowerCase() !== 'html') {
      var result = partForNode(node, doc);
      path.unshift(result.part);
      var candidate = path.join(' > ');
      if (isUnique(candidate, el, doc)) {
        return candidate;
      }
      if (result.anchored) {
        break;
      }
      node = node.parentElement;
    }

    return path.join(' > ');
  }

  // ---------------------------------------------------------------------------
  // XPath generation
  // ---------------------------------------------------------------------------

  function absSegment(el) {
    var tag = el.tagName.toLowerCase();
    return tag + '[' + nthOfTypeIndex(el) + ']';
  }

  function getAbsoluteXPath(el) {
    if (!el || el.nodeType !== 1) {
      return null;
    }
    var segs = [];
    var node = el;
    while (node && node.nodeType === 1) {
      segs.unshift(absSegment(node));
      node = node.parentElement;
    }
    return '/' + segs.join('/');
  }

  // Turn a string into a safe XPath literal, using concat() when it contains
  // both single and double quotes.
  function xpathLiteral(value) {
    var s = String(value);
    if (s.indexOf("'") === -1) {
      return "'" + s + "'";
    }
    if (s.indexOf('"') === -1) {
      return '"' + s + '"';
    }
    var pieces = s.split("'").map(function (piece) {
      return "'" + piece + "'";
    });
    return 'concat(' + pieces.join(', "\'", ') + ')';
  }

  function defaultEvaluator(doc) {
    // XPathResult lives on the document's window in the browser and on the
    // jsdom window in tests; fall back to a bare global just in case.
    var view = doc && doc.defaultView;
    var XPR = (view && view.XPathResult) ||
      (typeof XPathResult !== 'undefined' ? XPathResult : null);
    if (doc && typeof doc.evaluate === 'function' && XPR) {
      return function (expr) {
        try {
          var r = doc.evaluate(expr, doc, null, XPR.ORDERED_NODE_SNAPSHOT_TYPE, null);
          var out = [];
          for (var i = 0; i < r.snapshotLength; i++) {
            out.push(r.snapshotItem(i));
          }
          return out;
        } catch (e) {
          return null;
        }
      };
    }
    return null;
  }

  // A short, human-readable XPath keyed on an id, stable attribute, or leaf text
  // when one of those uniquely resolves; otherwise anchored to the nearest id
  // ancestor; otherwise the absolute path.
  function getRelativeXPath(el, options) {
    options = options || {};
    if (!el || el.nodeType !== 1) {
      return null;
    }
    var doc = el.ownerDocument || (typeof document !== 'undefined' ? document : null);
    var evaluate = options.evaluate || (doc ? defaultEvaluator(doc) : null);
    var tag = el.tagName.toLowerCase();

    var candidates = [];

    if (el.id && !looksDynamic(el.id)) {
      candidates.push('//' + tag + '[@id=' + xpathLiteral(el.id) + ']');
      candidates.push('//*[@id=' + xpathLiteral(el.id) + ']');
    }

    if (el.hasAttribute) {
      for (var i = 0; i < STABLE_ATTRS.length; i++) {
        var attr = STABLE_ATTRS[i];
        if (el.hasAttribute(attr)) {
          var val = el.getAttribute(attr);
          if (val && val.length <= MAX_ATTR_VALUE && !looksDynamic(val)) {
            candidates.push('//' + tag + '[@' + attr + '=' + xpathLiteral(val) + ']');
          }
        }
      }
    }

    var text = (el.textContent || '').replace(/\s+/g, ' ').trim();
    if (text && text.length <= MAX_XPATH_TEXT && el.children.length === 0) {
      candidates.push('//' + tag + '[normalize-space()=' + xpathLiteral(text) + ']');
    }

    if (evaluate) {
      for (var c = 0; c < candidates.length; c++) {
        var nodes = evaluate(candidates[c]);
        if (nodes && nodes.length === 1 && nodes[0] === el) {
          return candidates[c];
        }
      }
    } else if (candidates.length) {
      // No evaluator available: return the strongest guess unverified.
      return candidates[0];
    }

    // Anchor to the nearest ancestor that carries a stable id.
    var tail = [absSegment(el)];
    var anc = el.parentElement;
    while (anc && anc.nodeType === 1 && anc.tagName.toLowerCase() !== 'html') {
      if (anc.id && !looksDynamic(anc.id)) {
        var anchored = '//*[@id=' + xpathLiteral(anc.id) + ']/' + tail.join('/');
        if (!evaluate) {
          return anchored;
        }
        var res = evaluate(anchored);
        if (res && res.length === 1 && res[0] === el) {
          return anchored;
        }
      }
      tail.unshift(absSegment(anc));
      anc = anc.parentElement;
    }

    return getAbsoluteXPath(el);
  }

  // ---------------------------------------------------------------------------
  // Element inspection + Python snippet generation
  // ---------------------------------------------------------------------------

  function getElementInfo(el) {
    var attrs = {};
    if (el.attributes) {
      for (var i = 0; i < el.attributes.length; i++) {
        var a = el.attributes[i];
        attrs[a.name] = a.value;
      }
    }
    var text = (el.textContent || '').replace(/\s+/g, ' ').trim();
    return {
      tag: el.tagName ? el.tagName.toLowerCase() : null,
      id: el.id || null,
      classes: el.classList ? Array.prototype.slice.call(el.classList) : [],
      attributes: attrs,
      text: text.length > 140 ? text.slice(0, 140) + '…' : text
    };
  }

  // Render a Python string literal, preferring double quotes.
  function pyStr(value) {
    var s = String(value);
    if (s.indexOf('"') === -1) {
      return '"' + s.replace(/\\/g, '\\\\') + '"';
    }
    if (s.indexOf("'") === -1) {
      return "'" + s.replace(/\\/g, '\\\\') + "'";
    }
    return '"' + s.replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"';
  }

  var CREDIT = '# selector built with selector-picker — guides: https://python-web-scraping.com';

  function buildSnippets(sel) {
    sel = sel || {};
    var css = sel.css || '';
    var xpath = sel.xpath || sel.xpathAbsolute || '';
    var pyCss = pyStr(css);
    var pyXpath = pyStr(xpath);

    return {
      beautifulsoup: [
        CREDIT,
        'from bs4 import BeautifulSoup',
        '',
        'soup = BeautifulSoup(html, "html.parser")',
        'el = soup.select_one(' + pyCss + ')',
        'text = el.get_text(strip=True) if el else None'
      ].join('\n'),

      parsel: [
        CREDIT,
        'from parsel import Selector',
        '',
        'sel = Selector(text=html)',
        'el = sel.css(' + pyCss + ').get()',
        '# XPath alternative:',
        'el = sel.xpath(' + pyXpath + ').get()'
      ].join('\n'),

      scrapy: [
        CREDIT,
        '# inside a Scrapy spider callback (self, response):',
        'value = response.css(' + pyStr(css + '::text') + ').get()',
        '# or via XPath:',
        'value = response.xpath(' + pyStr(xpath + '/text()') + ').get()'
      ].join('\n')
    };
  }

  // Convenience: everything about an element in one call.
  function pick(el, options) {
    var css = getCssSelector(el);
    var xpath = getRelativeXPath(el, options);
    var xpathAbsolute = getAbsoluteXPath(el);
    var info = getElementInfo(el);
    var snippets = buildSnippets({ css: css, xpath: xpath, xpathAbsolute: xpathAbsolute });
    return {
      css: css,
      xpath: xpath,
      xpathAbsolute: xpathAbsolute,
      info: info,
      snippets: snippets
    };
  }

  return {
    getCssSelector: getCssSelector,
    getAbsoluteXPath: getAbsoluteXPath,
    getRelativeXPath: getRelativeXPath,
    getElementInfo: getElementInfo,
    buildSnippets: buildSnippets,
    pick: pick,
    // exposed for testing / advanced use
    escapeCss: escapeCss,
    looksDynamic: looksDynamic,
    xpathLiteral: xpathLiteral,
    CREDIT: CREDIT
  };
});