diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a19efb..f8abeec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ All notable changes to `odh-cli` are documented here. ## Unreleased +- `datasets guide` ranks catalogue matches by how many query terms hit instead + of requiring every word. Natural phrases like `roadworks on a state road` no + longer return nothing while the bare topic still matched; filler words reuse + the traffic search stopword list. `datasets search` keeps the stricter + all-terms filter. Refs #14. - `traffic search` no longer matches alphabetic terms inside other words. Searching `auer` used to hit every `Stützmauern` notice; terms now match at a word boundary (prefix still allowed, so cycle aliases like `radweg` keep diff --git a/internal/commands/datasets.go b/internal/commands/datasets.go index a8405ad..465ac55 100644 --- a/internal/commands/datasets.go +++ b/internal/commands/datasets.go @@ -90,7 +90,7 @@ into agent answers.`, } query := strings.Join(args, " ") entries := filterDatasetsByDomain(datasetCatalog(), guideDomain) - entries = filterDatasetsByQuery(entries, query) + entries = rankDatasetsByQuery(entries, query) if guideLimit > 0 && len(entries) > guideLimit { entries = entries[:guideLimit] } @@ -174,13 +174,7 @@ func filterDatasetsByQuery(entries []datasetEntry, query string) []datasetEntry terms := strings.Fields(strings.ToLower(query)) filtered := make([]datasetEntry, 0, len(entries)) for _, entry := range entries { - haystack := strings.ToLower(strings.Join(append([]string{ - entry.ID, - entry.Domain, - entry.API, - entry.Title, - entry.Description, - }, entry.Keywords...), " ")) + haystack := datasetEntryHaystack(entry) matched := true for _, term := range terms { if !strings.Contains(haystack, term) { @@ -195,6 +189,70 @@ func filterDatasetsByQuery(entries []datasetEntry, query string) []datasetEntry return filtered } +// rankDatasetsByQuery scores catalogue entries by how many query terms hit and +// returns them best-first. Terms that hit nothing are dropped rather than +// requiring every word — guide is the natural-language entry point, so extra +// sentence words must not erase a real match. datasets search keeps the stricter +// all-terms filter. +func rankDatasetsByQuery(entries []datasetEntry, query string) []datasetEntry { + terms := datasetGuideQueryTerms(query) + if len(terms) == 0 { + out := make([]datasetEntry, len(entries)) + copy(out, entries) + return out + } + type scored struct { + entry datasetEntry + score int + } + ranked := make([]scored, 0, len(entries)) + for _, entry := range entries { + haystack := datasetEntryHaystack(entry) + score := 0 + for _, term := range terms { + if strings.Contains(haystack, term) { + score++ + } + } + if score > 0 { + ranked = append(ranked, scored{entry: entry, score: score}) + } + } + sort.SliceStable(ranked, func(i, j int) bool { + if ranked[i].score != ranked[j].score { + return ranked[i].score > ranked[j].score + } + return ranked[i].entry.ID < ranked[j].entry.ID + }) + out := make([]datasetEntry, len(ranked)) + for i, item := range ranked { + out[i] = item.entry + } + return out +} + +func datasetGuideQueryTerms(query string) []string { + terms := strings.Fields(strings.ToLower(query)) + out := make([]string, 0, len(terms)) + for _, term := range terms { + if trafficSearchStopword(term) { + continue + } + out = append(out, term) + } + return out +} + +func datasetEntryHaystack(entry datasetEntry) string { + return strings.ToLower(strings.Join(append([]string{ + entry.ID, + entry.Domain, + entry.API, + entry.Title, + entry.Description, + }, entry.Keywords...), " ")) +} + func datasetGuideFor(entry datasetEntry) datasetGuideEntry { guidance := datasetGuideEntry{Dataset: entry} switch entry.ID { diff --git a/internal/commands/runner_test.go b/internal/commands/runner_test.go index 6b22a9a..2ce2896 100644 --- a/internal/commands/runner_test.go +++ b/internal/commands/runner_test.go @@ -279,6 +279,70 @@ func TestRunDatasetsGuideTable(t *testing.T) { } } +// Guide is the natural-language entry point: extra sentence words must rank +// matches, not erase them. "roadworks on a state road" used to return nothing +// while "roadworks" returned the traffic catalogue entry. +func TestRunDatasetsGuideRanksNaturalPhrasing(t *testing.T) { + runner := newTestRunner(t, nil) + phrases := []struct { + query string + want string + }{ + {"roadworks", "mobility.traffic-events"}, + {"roadworks on a state road", "mobility.traffic-events"}, + {"where can I park and see forecasts", "mobility.parking"}, + {"ev charging availability nearby", "mobility.charging"}, + } + for _, phrase := range phrases { + var stdout, stderr bytes.Buffer + code := runner.Run(context.Background(), []string{"datasets", "guide", phrase.query, "--limit", "3"}, &stdout, &stderr) + if code != 0 { + t.Fatalf("guide %q exit = %d, stderr = %s", phrase.query, code, stderr.String()) + } + var decoded struct { + Count int `json:"count"` + Matches []struct { + Dataset struct { + ID string `json:"id"` + } `json:"dataset"` + } `json:"matches"` + } + if err := json.Unmarshal(stdout.Bytes(), &decoded); err != nil { + t.Fatalf("guide %q invalid JSON: %v\n%s", phrase.query, err, stdout.String()) + } + if decoded.Count == 0 { + t.Fatalf("guide %q returned no matches", phrase.query) + } + found := false + for _, match := range decoded.Matches { + if match.Dataset.ID == phrase.want { + found = true + break + } + } + if !found { + t.Fatalf("guide %q missing %s in %s", phrase.query, phrase.want, stdout.String()) + } + } +} + +func TestRankDatasetsByQueryPrefersMoreTermHits(t *testing.T) { + entries := []datasetEntry{ + {ID: "a", Title: "roadworks only", Keywords: []string{"roadworks"}}, + {ID: "b", Title: "parking forecast", Keywords: []string{"parking", "forecast"}}, + {ID: "c", Title: "unrelated", Keywords: []string{"tourism"}}, + } + ranked := rankDatasetsByQuery(entries, "parking forecast near me") + if len(ranked) != 1 || ranked[0].ID != "b" { + t.Fatalf("expected only parking forecast ranked first, got %#v", ranked) + } + // Stopwords and unmatched filler must not wipe a single catalogue hit. + ranked = rankDatasetsByQuery(entries, "roadworks on a state road") + if len(ranked) != 1 || ranked[0].ID != "a" { + t.Fatalf("expected roadworks to survive filler words, got %#v", ranked) + } +} + func TestDatasetCatalogCommandStringsParse(t *testing.T) { seen := map[string]struct{}{} commands := make([]string, 0)