Fix shtab bash completions list choices on <TAB><TAB> for types that accept other values

CHANGELOG.rst

@@ -33,6 +33,10 @@ Fixed
 ^^^^^
 - Parsing a list took time quadratic in its number of items (`#975
   <https://github.com/mauvilsa/jsonargparse/pull/975>`__).
+- ``shtab`` bash completions not listing the choices on ``<TAB><TAB>`` when
+  nothing has been typed and the type accepts values other than the choices,
+  e.g. ``int | SomeEnum`` (`#976
+  <https://github.com/mauvilsa/jsonargparse/pull/976>`__).
 
 Changed
 ^^^^^^^

DOCUMENTATION.rst

@@ -3438,7 +3438,8 @@ user. Take for example the parser:
 
 The completion prints the type of the argument, how many options match, and then
 the matching choices. If only one option matches, the value is completed without
-printing guidance. For example:
+printing guidance, unless nothing has been typed and the type accepts values
+other than the choices. For example:
 
 .. code-block:: bash
 
@@ -3448,6 +3449,12 @@ printing guidance. For example:
     $ example.py --bool f<TAB>
     $ example.py --bool false
 
+.. note::
+
+    The guidance requires bash 4 or newer. With older versions, e.g. bash 3.2 in
+    macOS, no guidance is printed, and the choices of types that accept other
+    values, like ``int | None``, are only completed after typing a prefix.
+
 For subclass types, the import paths of the known subclasses are completed, both
 for the option that selects the class and for the ``--*.help`` option. The
 ``init_args`` of the known subclasses are completed too, with guidance saying

jsonargparse/_completions.py

@@ -270,7 +270,13 @@ def shtab_prepare_action(action, parser) -> None:
   local CHOICES="$1" WORD="$2" MESSAGE="$3" REQUIRE_PREFIX="$4" TOTAL="$5"
   local IFS=$'\\n'  # choices may contain spaces, so split matches on newline only
   local MATCH=()
+  # when the choices are not all that is accepted, without a prefix a single TAB (COMP_TYPE=9)
+  # must not insert a choice, but <TAB><TAB> (COMP_TYPE=63) should still list them
+  local LIST_ONLY=0
   if [ "$REQUIRE_PREFIX" = 1 ] && [ -z "$WORD" ]; then
+    LIST_ONLY=1
+  fi
+  if [ "$LIST_ONLY" = 1 ] && [ "$COMP_TYPE" != 63 ]; then
     MATCH=()
   else
     MATCH=( $(IFS=" " compgen -W "$CHOICES" "$WORD") )
@@ -288,6 +294,10 @@ def shtab_prepare_action(action, parser) -> None:
     for match in "${MATCH[@]}"; do
       echo "$match"
     done
+    # bash inserts a single completion even on <TAB><TAB>, an extra empty one makes it only list
+    if [ "$LIST_ONLY" = 1 ] && [ ${#MATCH[@]} = 1 ]; then
+      echo ""
+    fi
     if [ "$COMP_TYPE" = 63 ]; then
       printf "${%(b)s}\\n%%s%%s${%(n)s}" "$MESSAGE" "$MATCHED" >&2
     fi

jsonargparse_tests/test_shtab.py

@@ -8,7 +8,7 @@
 import sys
 import tempfile
 import time
-from contextlib import suppress
+from contextlib import contextmanager, suppress
 from enum import Enum
 from importlib.util import find_spec
 from os import PathLike
@@ -135,12 +135,12 @@ def test_bash_object(parser, subtests):
 def test_bash_union_literal_and_any(parser, any_type, subtests):
     typehint = Union[Literal["one", "two"], any_type]
     parser.add_argument("--union", type=typehint)
-    # the choices are not all that is accepted, so a prefix is required to complete them
+    # the choices are not all that is accepted, so they are only listed when there is no prefix
     assert_bash_typehint_completions(
         subtests,
         parser,
         [
-            ("union", typehint, "", [], None),
+            ("union", typehint, "", ["one", "two"], None),
             ("union", typehint, "t", ["two"], "1/2"),
         ],
     )
@@ -176,7 +176,8 @@ def test_bash_optional_int(parser, subtests):
         subtests,
         parser,
         [
-            ("num", Optional[int], "", [], "0/1"),
+            # an extra empty completion makes bash list a single choice instead of inserting it
+            ("num", Optional[int], "", ["null", ""], "1/1"),
             ("num", Optional[int], "n", ["null"], "1/1"),
         ],
     )
@@ -287,7 +288,7 @@ def test_bash_union_literal_and_int(parser, subtests):
         subtests,
         parser,
         [
-            ("union", typehint, "", [], "0/1"),
+            ("union", typehint, "", ["false", ""], "1/1"),
             ("union", typehint, "f", ["false"], "1/1"),
         ],
     )
@@ -300,7 +301,7 @@ def test_bash_union_float_and_enum(parser, subtests):
         subtests,
         parser,
         [
-            ("union", typehint, "", [], "0/3"),
+            ("union", typehint, "", ["ABC", "XY", "XZ"], "3/3"),
             ("union", typehint, "X", ["XY", "XZ"], "2/3"),
         ],
     )
@@ -345,15 +346,25 @@ def test_bash_script_binds_redraw_current_line(parser):
     assert "bind '\"\\e[0n\": redraw-current-line'" in shtab_script
 
 
-def run_bash_typehint_completion(shtab_script, tmp_path, dest, word="", prefix=""):
+def run_bash_typehint_completion(shtab_script, tmp_path, dest, word="", prefix="", comp_type=63):
     shtab_script_path = tmp_path / "comp.sh"
     shtab_script_path.write_text(shtab_script)
-    sh = f'{prefix}source {shtab_script_path}; COMP_TYPE=63 _jsonargparse_tool_{dest}_typehint "{word}"'
+    sh = f'{prefix}source {shtab_script_path}; COMP_TYPE={comp_type} _jsonargparse_tool_{dest}_typehint "{word}"'
     popen = subprocess.Popen(["bash", "-c", sh], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     out, err = popen.communicate()
     return out.decode(), err.decode()
 
 
+@pytest.mark.parametrize("word", ["", "n"])
+def test_bash_single_tab_requires_prefix_for_union_with_open_values(parser, tmp_path, word):
+    parser.add_argument("--num", type=Optional[int])
+    shtab_script = get_shtab_script(parser, "bash")
+    # COMP_TYPE=9 is a single TAB, which would insert a single choice
+    out, err = run_bash_typehint_completion(shtab_script, tmp_path, "num", word=word, comp_type=9)
+    assert out.splitlines() == ([] if word == "" else ["null"])
+    assert err == ""
+
+
 def get_bash_tput_color():
     out = subprocess.run(["bash", "-c", "tput setaf 5 2>/dev/null"], capture_output=True)
     return out.stdout.decode()
@@ -401,20 +412,18 @@ def read_from_pty_until(fd, pattern, timeout=10.0):
     return out
 
 
-@pytest.mark.skipif(sys.platform == "win32", reason="pty is not available on Windows")
-@pytest.mark.filterwarnings("ignore:.*multi-threaded, use of forkpty.*:DeprecationWarning")
-def test_bash_interactive_no_completions_redraws_prompt(parser, tmp_path):
+@contextmanager
+def interactive_bash(parser, tmp_path, rc_lines=()):
     if get_bash_major_version() < 4:
         pytest.skip("test requires bash>=4")  # pragma: no cover
     import fcntl
     import pty
     import termios
 
-    parser.add_argument("--num", type=int)
     shtab_script_path = tmp_path / "comp.sh"
     shtab_script_path.write_text(get_shtab_script(parser, "bash"))
     rcfile = tmp_path / "rcfile"
-    rcfile.write_text(f"PS1='PROMPT$ '\nsource {shtab_script_path}\n")
+    rcfile.write_text("\n".join(["PS1='PROMPT$ '", f"source {shtab_script_path}", *rc_lines, ""]))
 
     pid, fd = pty.fork()
     if pid == 0:  # pragma: no cover
@@ -426,13 +435,7 @@ def test_bash_interactive_no_completions_redraws_prompt(parser, tmp_path):
     try:
         fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", 40, 200, 0, 0))
         read_from_pty_until(fd, b"PROMPT$ ")
-        os.write(fd, b"tool --num \t\t")
-        out = read_from_pty_until(fd, b"\x1b[5n")
-        assert b"Expected type: int" in out
-        assert b"\x1b[5n" in out, "completion should request a device status report from the terminal"
-        os.write(fd, b"\x1b[0n")  # a real terminal replies this to the \x1b[5n device status report
-        out = read_from_pty_until(fd, b"PROMPT$ tool --num ")
-        assert b"PROMPT$ tool --num " in out, "prompt should be redrawn after the guidance message"
+        yield fd
     finally:
         with suppress(OSError):
             os.write(fd, b"\x03exit\n")
@@ -442,6 +445,36 @@ def test_bash_interactive_no_completions_redraws_prompt(parser, tmp_path):
             os.waitpid(pid, 0)
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="pty is not available on Windows")
+@pytest.mark.filterwarnings("ignore:.*multi-threaded, use of forkpty.*:DeprecationWarning")
+def test_bash_interactive_no_completions_redraws_prompt(parser, tmp_path):
+    parser.add_argument("--num", type=int)
+    with interactive_bash(parser, tmp_path) as fd:
+        os.write(fd, b"tool --num \t\t")
+        out = read_from_pty_until(fd, b"\x1b[5n")
+        assert b"Expected type: int" in out
+        assert b"\x1b[5n" in out, "completion should request a device status report from the terminal"
+        os.write(fd, b"\x1b[0n")  # a real terminal replies this to the \x1b[5n device status report
+        out = read_from_pty_until(fd, b"PROMPT$ tool --num ")
+        assert b"PROMPT$ tool --num " in out, "prompt should be redrawn after the guidance message"
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="pty is not available on Windows")
+@pytest.mark.filterwarnings("ignore:.*multi-threaded, use of forkpty.*:DeprecationWarning")
+def test_bash_interactive_lists_single_choice_without_inserting(parser, tmp_path):
+    parser.add_argument("--union", type=Union[Literal[False], int])
+    # Ctrl-T prints the current command line, to check what completion inserted
+    rc_lines = ['bind -x \'"\\C-t": printf "LINE=[%s]\\n" "$READLINE_LINE"\'']
+    with interactive_bash(parser, tmp_path, rc_lines) as fd:
+        os.write(fd, b"tool --union \t\t")
+        out = read_from_pty_until(fd, b"1/1 matched choices")
+        assert b"1/1 matched choices" in out, "the choice should be matched on <TAB><TAB>"
+        os.write(fd, b"\x14")
+        out += read_from_pty_until(fd, b"]\r\n")
+        assert b"false" in out, "the choice should be listed"
+        assert b"LINE=[tool --union ]" in out, "the choice should not be inserted"
+
+
 def test_bash_config(parser):
     parser.add_argument("--cfg", action="config")
     shtab_script = get_shtab_script(parser, "bash")

pyproject.toml

@@ -69,8 +69,7 @@ fsspec = [
     "fsspec>=0.8.4",
 ]
 shtab = [
-    # 1.8.2 and 1.9.0 generate corrupted *_COMPGEN values, fixed in 1.9.1
-    "shtab>=1.7.1,!=1.8.2,!=1.9.0",
+    "shtab>=1.9.1",
 ]
 argcomplete = [
     "argcomplete>=3.5.1",