diff --git a/cmd/openai/main_suggest_test.go b/cmd/openai/main_suggest_test.go new file mode 100644 index 00000000..35cbb3a2 --- /dev/null +++ b/cmd/openai/main_suggest_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "strings" + "testing" +) + +// Unknown commands suggest the full path of a close match and stay silent +// when nothing is close. +func TestMainUnknownCommandSuggestions(t *testing.T) { + for _, tc := range []struct { + args []string + stderr string + }{ + {[]string{"fine-tuning:alpha:graders", "rn"}, "No help topic for 'rn'. Did you mean 'openai fine-tuning:alpha:graders run'?\n"}, + {[]string{"fine-tuning:alpha:graders", "rnu"}, "No help topic for 'rnu'. Did you mean 'openai fine-tuning:alpha:graders run'?\n"}, + {[]string{"responses", "creat"}, "No help topic for 'creat'. Did you mean 'openai responses create'?\n"}, + {[]string{"RESPONSES"}, "No help topic for 'RESPONSES'. Did you mean 'openai responses'?\n"}, + {[]string{"totallybogus"}, "No help topic for 'totallybogus'\n"}, + {[]string{"responses", "zzzzz"}, "No help topic for 'zzzzz'\n"}, + } { + t.Run(strings.Join(tc.args, "/"), func(t *testing.T) { + want := mainDispatchResult{3, "", tc.stderr} + if got := runMainDispatch(t, "bash", append([]string{"openai"}, tc.args...)...); got != want { + t.Fatalf("got %+v; want %+v", got, want) + } + }) + } +} diff --git a/pkg/cmd/suggest.go b/pkg/cmd/suggest.go index b4b637c0..a2af6859 100644 --- a/pkg/cmd/suggest.go +++ b/pkg/cmd/suggest.go @@ -10,7 +10,7 @@ import ( ) // This entire file is mostly taken from urfave/cli/v3's source, with the exception of suggestCommand which is -// modified for a nicer error message. +// modified for a nicer error message and stricter matching, and its helpers suggestionThreshold and withinOneEdit. // jaroDistance is the measure of similarity between two strings. It returns a // value between 0 and 1, where 1 indicates identical strings and 0 indicates @@ -98,20 +98,69 @@ func jaroWinkler(a, b string) float64 { return jaroDist + 0.1*prefixMatch*(1.0-jaroDist) } +// suggestionThreshold is the minimum jaro-winkler similarity required before we +// will print a "Did you mean" suggestion. Below this, the closest match is too +// dissimilar to be a useful guess, so we stay silent rather than mislead. +// 0.7 matches the boostThreshold used by jaroWinkler above — the prefix boost +// only kicks in past that, so it's a natural "plausibly the same word" cutoff. +// Names within one edit of the input are exempt; see suggestCommand. +const suggestionThreshold = 0.7 + +// withinOneEdit reports whether a and b differ by at most one insertion, +// deletion, substitution, or transposition of adjacent characters. +func withinOneEdit(a, b string) bool { + if len(a) < len(b) { + a, b = b, a + } + if len(a)-len(b) > 1 { + return false + } + i := 0 + for i < len(b) && a[i] == b[i] { + i++ + } + if i == len(a) { + return true + } + if len(a) != len(b) { + return a[i+1:] == b[i:] + } + return a[i+1:] == b[i+1:] || + (i+1 < len(a) && a[i] == b[i+1] && a[i+1] == b[i] && a[i+2:] == b[i+2:]) +} + // suggestCommand takes a list of commands and a provided string to suggest a -// command name +// command name, ignoring case. A name within one edit of the input is always +// suggested and outranks every other name, because jaro-winkler undervalues +// typos in short names ("rn" for "run") and misranks long names that share a +// prefix. Otherwise the closest name must exceed suggestionThreshold. Returns an +// empty string when no command is sufficiently similar; the upstream urfave/cli +// error formatter omits the suggestion clause in that case. func suggestCommand(commands []*cli.Command, provided string) string { - distance := 0.0 + provided = strings.ToLower(provided) + if provided == "" { + return "" + } + // An exact 7/10 jaro score computes as 0.7000000000000001, so allow for + // float error before counting the threshold as passed. + distance := suggestionThreshold + 1e-9 var lineage []*cli.Command for _, command := range commands { for _, name := range command.Names() { + name = strings.ToLower(name) newDistance := jaroWinkler(name, provided) + if withinOneEdit(name, provided) { + newDistance++ // outranks every name further away + } if newDistance > distance { distance = newDistance lineage = command.Lineage() } } } + if lineage == nil { + return "" + } var parts []string for _, command := range lineage { diff --git a/pkg/cmd/suggest_test.go b/pkg/cmd/suggest_test.go new file mode 100644 index 00000000..4ee51537 --- /dev/null +++ b/pkg/cmd/suggest_test.go @@ -0,0 +1,205 @@ +package cmd + +import ( + "fmt" + "strings" + "testing" + + "github.com/urfave/cli/v3" +) + +func TestSuggestCommand(t *testing.T) { + commands := []*cli.Command{ + {Name: "create"}, + {Name: "retrieve"}, + {Name: "list"}, + {Name: "delete"}, + {Name: "chat:completions"}, + {Name: "completions"}, + } + + tests := []struct { + name string + provided string + want string + }{ + { + name: "close typo suggests the corrected command", + provided: "creat", + want: "Did you mean 'create'?", + }, + { + name: "near-exact suggests the corrected command", + provided: "chat:completion", + want: "Did you mean 'chat:completions'?", + }, + { + name: "exact match still suggests itself", + provided: "create", + want: "Did you mean 'create'?", + }, + { + name: "uppercase input is matched ignoring case", + provided: "CREAT", + want: "Did you mean 'create'?", + }, + { + name: "unrelated input returns no suggestion", + provided: "zzzzz", + want: "", + }, + { + name: "low-similarity input returns no suggestion", + provided: "totallybogus", + want: "", + }, + { + name: "empty input returns no suggestion", + provided: "", + want: "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := suggestCommand(commands, tc.provided) + if got != tc.want { + t.Errorf("suggestCommand(%q) = %q, want %q", tc.provided, got, tc.want) + } + }) + } +} + +func TestSuggestCommandEmptyCommands(t *testing.T) { + if got := suggestCommand(nil, "anything"); got != "" { + t.Errorf("suggestCommand(nil, %q) = %q, want empty string", "anything", got) + } +} + +func TestWithinOneEdit(t *testing.T) { + tests := []struct { + a, b string + want bool + }{ + {"", "", true}, + {"", "a", true}, + {"", "ab", false}, + {"run", "run", true}, + {"run", "rn", true}, // deletion + {"run", "runn", true}, // insertion + {"run", "ran", true}, // substitution + {"run", "rnu", true}, // transposition + {"run", "urn", true}, // transposition + {"ab", "ba", true}, // transposition + {"run", "rm", false}, + {"run", "nur", false}, + {"list", "ls", false}, + {"create", "craete", true}, + {"create", "carete", false}, + {"create", "caerte", false}, + {"run", "urm", false}, // swapped pair plus another change + {"run", "xrn", false}, // only one side of the swap matches + } + for _, tc := range tests { + for _, pair := range [][2]string{{tc.a, tc.b}, {tc.b, tc.a}} { + if got := withinOneEdit(pair[0], pair[1]); got != tc.want { + t.Errorf("withinOneEdit(%q, %q) = %v, want %v", pair[0], pair[1], got, tc.want) + } + } + } +} + +// findCommand resolves a command path in the real command tree. +func findCommand(t *testing.T, path ...string) *cli.Command { + t.Helper() + command := Command + for _, name := range path { + var next *cli.Command + for _, sub := range command.Commands { + if sub.Name == name { + next = sub + } + } + if next == nil { + t.Fatalf("command %q not found under %q", name, command.Name) + } + command = next + } + return command +} + +// Parent links are only set once the command tree runs, so these suggestions +// name the matched command alone; cmd/openai checks the full "openai ..." form. +func TestSuggestCommandRealTree(t *testing.T) { + tests := []struct { + path []string + provided string + want string + }{ + {[]string{"fine-tuning:alpha:graders"}, "rn", "run"}, + {[]string{"fine-tuning:alpha:graders"}, "rnu", "run"}, + {[]string{"fine-tuning:alpha:graders"}, "urn", "run"}, + {[]string{"fine-tuning:alpha:graders"}, "RN", "run"}, + {[]string{"fine-tuning:alpha:graders"}, "valdate", "validate"}, + {[]string{"fine-tuning:alpha:graders"}, "zzz", ""}, + {[]string{"responses"}, "zzzzz", ""}, + {[]string{"models"}, "ls", "list"}, // jaro-winkler 0.85 + {[]string{"models"}, "lete", "delete"}, // jaro-winkler 0.72 + // Both are one edit away; the higher jaro-winkler score wins. + {[]string{"admin:organization:certificates"}, "dactivate", "deactivate"}, + // Exact 7/10 jaro scores that float error would otherwise let through. + {[]string{"fine-tuning:jobs"}, "status", ""}, + {nil, "hi", ""}, + {nil, "RESPONSES", "responses"}, + {nil, "respones", "responses"}, + {nil, "chat:completion", "chat:completions"}, + {nil, "comp", "completions"}, + // Jaro-winkler alone prefers admin:organization:spend-alerts here. + {nil, "admin:prganization:roles", "admin:organization:roles"}, + {nil, "version", ""}, // jaro-winkler 0.69 + {nil, "totallybogus", ""}, + } + for _, tc := range tests { + t.Run(strings.Join(append(tc.path, tc.provided), " "), func(t *testing.T) { + want := "" + if tc.want != "" { + want = fmt.Sprintf("Did you mean '%s'?", tc.want) + } + if got := suggestCommand(findCommand(t, tc.path...).Commands, tc.provided); got != want { + t.Errorf("got %q, want %q", got, want) + } + }) + } +} + +// Every command in the real tree keeps its suggestion for a dropped or swapped +// character or for all-caps input, unless the typo is as close to a sibling. +func TestSuggestCommandRealTreeSingleEdits(t *testing.T) { + var visit func(parent *cli.Command) + visit = func(parent *cli.Command) { + for _, command := range parent.Commands { + name := command.Name + typos := []string{strings.ToUpper(name)} + for i := range name { + typos = append(typos, name[:i]+name[i+1:]) + if i+1 < len(name) { + typos = append(typos, name[:i]+name[i+1:i+2]+name[i:i+1]+name[i+2:]) + } + } + typos: + for _, typo := range typos { + for _, sibling := range parent.Commands { + if sibling != command && withinOneEdit(sibling.Name, strings.ToLower(typo)) { + continue typos + } + } + want := fmt.Sprintf("Did you mean '%s'?", name) + if got := suggestCommand(parent.Commands, typo); got != want { + t.Errorf("%s: suggestCommand(%q) = %q, want %q", parent.Name, typo, got, want) + } + } + visit(command) + } + } + visit(Command) +}