Keep smart quotes off --option mentions in help text

sphinxarg/ext.py

@@ -3,6 +3,7 @@
 import importlib
 import operator
 import os
+import re
 import sys
 from argparse import ArgumentParser
 from typing import TYPE_CHECKING, cast
@@ -17,7 +18,7 @@
 from sphinx.roles import XRefRole
 from sphinx.util import logging
 from sphinx.util.docutils import SphinxDirective, new_document
-from sphinx.util.nodes import make_id, make_refnode, nested_parse_with_titles
+from sphinx.util.nodes import is_smartquotable, make_id, make_refnode, nested_parse_with_titles
 
 try:
     from sphinx.ext.autodoc.mock import mock
@@ -33,7 +34,7 @@
     from collections.abc import Iterable, Sequence
     from pathlib import Path
 
-    from docutils.nodes import Element
+    from docutils.nodes import Element, Node
     from sphinx.addnodes import pending_xref
     from sphinx.application import Sphinx
     from sphinx.builders import Builder
@@ -88,6 +89,49 @@ def map_nested_definitions(nested_content):
     return definitions
 
 
+_OPTION_TOKEN = re.compile(r'((?<!\w)--[a-zA-Z0-9][\w-]*)')
+
+
+def _protect_option_dashes(children: list[Node]) -> None:
+    """Shield ``--option`` tokens from smart quotes rewriting ``--`` as an en dash.
+
+    Mutates the tree under ``children`` in place. Each token is wrapped in an
+    unstyled inline exempted via ``support_smartquotes``.
+    """
+    for child in children:
+        if not isinstance(child, nodes.Element):
+            continue
+        for text in list(child.findall(nodes.Text)):
+            parent = text.parent
+            # The same check the SmartQuotes transform uses: skips text that is
+            # already exempt (literals, literal blocks, ...) at any depth.
+            if not is_smartquotable(text):
+                continue
+            # Capturing group => split() yields the option tokens at odd
+            # indices, interleaved with the surrounding text.
+            parts = _OPTION_TOKEN.split(text)
+            if len(parts) == 1:
+                continue
+            replacement: list[Node] = []
+            for i, part in enumerate(parts):
+                if not part:
+                    continue
+                if i % 2:
+                    inline = nodes.inline('', part)
+                    inline['support_smartquotes'] = False
+                    replacement.append(inline)
+                else:
+                    replacement.append(nodes.Text(part))
+            parent.replace(text, replacement)
+
+
+def _option_safe_paragraph(text: str) -> nodes.paragraph:
+    """Return a paragraph of parser-supplied text with option dashes protected."""
+    paragraph = nodes.paragraph(text=text)
+    _protect_option_dashes([paragraph])
+    return paragraph
+
+
 def render_list(l, markdown_help, settings=None):
     """
     Given a list of reStructuredText or MarkDown sections, return a docutils node list
@@ -97,7 +141,9 @@ def render_list(l, markdown_help, settings=None):
     if markdown_help:
         from sphinxarg.markdown import parse_markdown_block
 
-        return parse_markdown_block('\n\n'.join(l) + '\n')
+        blocks = parse_markdown_block('\n\n'.join(l) + '\n')
+        _protect_option_dashes(blocks)
+        return blocks
     else:
         if settings is None:
             settings = get_default_settings(Parser)
@@ -106,6 +152,7 @@ def render_list(l, markdown_help, settings=None):
             if isinstance(element, str):
                 document = new_document('', settings)
                 Parser().parse(element + '\n', document)
+                _protect_option_dashes(document.children)
                 all_children += document.children
             elif isinstance(element, nodes.definition):
                 all_children += element
@@ -356,8 +403,8 @@ def _construct_manpage_specific_structure(self, parser_info):
             description_section = nodes.section(
                 '',
                 nodes.title(text='Description'),
-                nodes.paragraph(
-                    text=parser_info.get(
+                _option_safe_paragraph(
+                    parser_info.get(
                         'description',
                         parser_info.get('help', 'undocumented').capitalize(),
                     )
@@ -372,9 +419,9 @@ def _construct_manpage_specific_structure(self, parser_info):
             # parse method invoked above seem to be able to do this but
             # I haven't found a way to do it for arbitrary text
             if description_section:
-                description_section += nodes.paragraph(text=parser_info['epilog'])
+                description_section += _option_safe_paragraph(parser_info['epilog'])
             else:
-                description_section = nodes.paragraph(text=parser_info['epilog'])
+                description_section = _option_safe_paragraph(parser_info['epilog'])
                 items.append(description_section)
         # OPTIONS section
         options_section = nodes.section(
@@ -426,12 +473,12 @@ def _format_positional_arguments(self, parser_info):
         for arg in parser_info['args']:
             arg_items = []
             if arg['help']:
-                arg_items.append(nodes.paragraph(text=arg['help']))
+                arg_items.append(_option_safe_paragraph(arg['help']))
             elif 'choices' not in arg:
                 arg_items.append(nodes.paragraph(text='Undocumented'))
             if 'choices' in arg:
                 arg_items.append(
-                    nodes.paragraph(text='Possible choices: ' + ', '.join(arg['choices']))
+                    _option_safe_paragraph('Possible choices: ' + ', '.join(arg['choices']))
                 )
             items.append(
                 nodes.option_list_item(
@@ -458,12 +505,12 @@ def _format_optional_arguments(self, parser_info):
                     )
                 names.append(nodes.option('', *option_declaration))
             if opt['help']:
-                opt_items.append(nodes.paragraph(text=opt['help']))
+                opt_items.append(_option_safe_paragraph(opt['help']))
             elif 'choices' not in opt:
                 opt_items.append(nodes.paragraph(text='Undocumented'))
             if 'choices' in opt:
                 opt_items.append(
-                    nodes.paragraph(text='Possible choices: ' + ', '.join(opt['choices']))
+                    _option_safe_paragraph('Possible choices: ' + ', '.join(opt['choices']))
                 )
             items.append(
                 nodes.option_list_item(
@@ -480,13 +527,15 @@ def _format_subcommands(self, parser_info):
         for subcmd in parser_info['children']:
             subcmd_items = []
             if subcmd['help']:
-                subcmd_items.append(nodes.paragraph(text=subcmd['help']))
+                subcmd_items.append(_option_safe_paragraph(subcmd['help']))
             else:
                 subcmd_items.append(nodes.paragraph(text='Undocumented'))
+            # The usage line is command-line syntax, so no smart quotes at all.
+            usage = nodes.strong(text=subcmd['bare_usage'], support_smartquotes=False)
             items.append(
                 nodes.definition_list_item(
                     '',
-                    nodes.term('', '', nodes.strong(text=subcmd['bare_usage'])),
+                    nodes.term('', '', usage),
                     nodes.definition('', *subcmd_items),
                 )
             )
@@ -495,6 +544,7 @@ def _format_subcommands(self, parser_info):
     def _nested_parse_paragraph(self, text):
         content = nodes.paragraph()
         self.state.nested_parse(StringList(text.split('\n')), 0, content)
+        _protect_option_dashes([content])
         return content
 
     @property

test/conftest.py

@@ -71,7 +71,7 @@ def cached_etree_parse():
     def parse(fname):
         if fname in etree_cache:
             return etree_cache[fname]
-        with (fname).open('r') as fp:
+        with (fname).open('r', encoding='utf-8') as fp:
             data = fp.read().replace('\n', '')
             etree = lxmltree.HTML(data)
             etree_cache.clear()

test/roots/test-default-html/smartquotes-manpage.rst

@@ -0,0 +1,7 @@
+Smart quotes in manpage mode
+============================
+
+.. argparse::
+   :module: test.sample_smartquotes
+   :func: get_parser
+   :manpage:

test/roots/test-default-html/smartquotes.rst

@@ -0,0 +1,8 @@
+Smart quotes
+============
+
+.. argparse::
+   :module: test.sample_smartquotes
+   :func: get_parser
+
+   Directive body prose keeps typography -- an en dash.

test/sample_smartquotes.py

@@ -0,0 +1,34 @@
+import argparse
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(
+        prog='sample-smartquotes',
+        description='Read --input and write the result to --output.',
+        epilog=(
+            '--verbose can be repeated. Quotes "like these", a dash -- like this -- '
+            'and east--west keep their typography.'
+        ),
+    )
+    parser.add_argument(
+        'mode',
+        choices=['copy', 'move'],
+        help='What to do with the --input file.',
+    )
+    parser.add_argument('--input', help='Source file; see also --output.')
+    parser.add_argument(
+        '--output',
+        help='Target file. Markup such as ``--mode`` is left as it is.',
+    )
+    parser.add_argument(
+        '--level',
+        choices=['low', 'high'],
+        help='Verbosity, with no option mentioned.',
+    )
+    parser.add_argument('--verbose', action='store_true', help='Say more.')
+
+    subparsers = parser.add_subparsers(dest='command')
+    run = subparsers.add_parser('run', help='Run it, honouring --input.')
+    run.add_argument('--fast', action='store_true', help='Skip checks; implies --quiet.')
+
+    return parser

test/test_default_html.py

@@ -71,6 +71,51 @@
                 ('.//section/dl/dd/p', 'Default', False),
             ],
         ),
+        (
+            'smartquotes.html',
+            [
+                ('.//h1', 'Smart quotes'),
+                # Option mentions in the description, help, epilog and sub-command
+                # help keep both hyphens instead of becoming an en dash (#100)...
+                ('.//p/span', '--input'),
+                ('.//p/span', '--output'),
+                ('.//p/span', '--verbose'),
+                ('.//p/span', '--quiet'),
+                (".//*[contains(., '–input')]", None),
+                (".//*[contains(., '–output')]", None),
+                (".//*[contains(., '–verbose')]", None),
+                (".//*[contains(., '–quiet')]", None),
+                # ...inline literals are not wrapped a second time...
+                ('.//p/code/span', '--mode'),
+                ('.//p/code/span/span', None),
+                # ...and the surrounding prose keeps its typography.
+                (".//p[contains(., '“like these”')]", ''),
+                (".//p[contains(., 'dash – like this –')]", ''),
+                (".//p[contains(., 'east–west')]", ''),
+                # Text written in the directive body is reST and is left alone.
+                ('.//p', 'keeps typography – an en dash'),
+            ],
+        ),
+        (
+            'smartquotes-manpage.html',
+            [
+                ('.//h1', 'Smart quotes in manpage mode'),
+                ('.//h2', 'Description'),
+                ('.//section/p/span', '--input'),
+                ('.//section/p/span', '--verbose'),
+                ('.//dd/p/span', '--output'),
+                (".//dd/p[contains(., 'honouring')]/span", '--input'),
+                ('.//dd/p', 'Possible choices: copy, move'),
+                # The sub-command usage line is command-line syntax throughout.
+                ('.//dt/strong', r'run \[-h\] \[--fast\]'),
+                (".//*[contains(., '–input')]", None),
+                (".//*[contains(., '–output')]", None),
+                (".//*[contains(., '–verbose')]", None),
+                (".//*[contains(., '–fast')]", None),
+                (".//p[contains(., '“like these”')]", ''),
+                (".//p[contains(., 'east–west')]", ''),
+            ],
+        ),
     ],
 )
 @pytest.mark.sphinx('html', testroot='default-html')