diff --git a/.gitignore b/.gitignore index 453c8bef..e3ad1d91 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,7 @@ docs/* !docs/search_ai_mode.md !docs/search_retrieve_link_content.md !docs/search_ai_overview_research.md +!docs/search_result_scoring.md .DS_Store # Spyder project settings diff --git a/README.md b/README.md index a0fb6ff1..52de3d6e 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,9 @@ Recipes are written in YAML and allow a series of Wrangles to be run as an autom For targeted web content retrieval with prompts that use values from each row, see [Retrieve content from links](docs/search_retrieve_link_content.md). +For part-code match evidence and counts in scored search results, see +[Search-result part-code evidence](docs/search_result_scoring.md). + Recipes can be triggered either from python code or a terminal command. #### Run ```python diff --git a/docs/search_result_scoring.md b/docs/search_result_scoring.md new file mode 100644 index 00000000..19bed1ac --- /dev/null +++ b/docs/search_result_scoring.md @@ -0,0 +1,79 @@ +# Search-result part-code evidence + +`compute.score_search_results` scores classic search payloads and adds two +diagnostic fields to each scored-result dictionary: + +- `part_code_matches`: reduced evidence from the result's title, snippet, and URL. +- `part_code_match_count`: the number of distinct candidate/type/source matches + collected before redundancy removal. + +These fields are nested alongside `summary`, `pricing`, `scoring_details`, and +`metadata`. They do not add a third recipe output. A single output column still +receives scored dictionaries; two output columns still receive dictionaries and +formatted summaries. The numeric scores, filtering, and result ordering are +unchanged. + +## Evidence records + +The keys in each record appear in this order: + +| Key | Values or meaning | +| --- | --- | +| `match_type` | `MPN` for an MPN input, or `Codes` for a `part_codes` input. | +| `match_level` | `exact`, `stripped`, or `partial`. | +| `match_source` | `title`, `snippet`, or `url`. | +| `matched_code` | The matching text from the result, preserving its case and separators. | +| `input_code` | The input candidate, with surrounding whitespace removed. | + +Matching is case-insensitive. `exact` means the complete source code matches the +input apart from case. `stripped` means the codes match after normalization with +the existing alphanumeric helper, which removes punctuation and spaces and folds +international characters. `partial` means the normalized input is contained in +a larger source code. Partial matching requires at least four normalized input +characters to avoid incidental matches for short codes. + +For example, `NATV6-PP-A` against `natv6-pp-a` is exact, `LBBR14-2LS` against +`LBBR 14-2LS` is stripped, and `NATV6` against `NATV6-X-PP-A` is partial. A +standalone `085-196-225` followed by sentence punctuation remains exact. + +The diagnostic levels describe the source evidence more precisely than the +existing scoring labels. In particular, the existing scoring evaluator can +award its exact-match score to normalized embedded text; this does not turn a +diagnostic partial match into an exact one. + +## Redundancy removal and counts + +Each result contains zero or one MPN record, placed before all Code records even +when the MPN match is partial. The strongest MPN evidence wins: exact before +stripped before partial, then the longer normalized input, then title before +snippet before URL. + +Code records are removed when they repeat an MPN input that matched, their +normalized input is contained in the selected MPN input, or their normalized +matched text is contained in the selected MPN matched text. Remaining records +with the same normalized matched code are collapsed across sources, using the +same strength, input-length, and source preferences. + +`part_code_match_count` is recorded before these reductions. It includes the MPN +repeated in `part_codes` and evidence from multiple sources, but does not count +identical duplicate candidates or repeated occurrences within one source. It is +an evidence count, not a count of distinct products or codes. No matches produces +an empty list and a count of zero. + +For an MPN of `NATV6-PP-A`, `part_codes` of `[NATV6-PP-A, NATV6]`, and a title of +`INA NATV6-PP-A track roller` with no other matching fields, the result includes: + +```json +{ + "part_code_matches": [ + { + "match_type": "MPN", + "match_level": "exact", + "match_source": "title", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6-PP-A" + } + ], + "part_code_match_count": 3 +} +``` diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index c0d5ade5..96699df8 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -1,5 +1,7 @@ import wrangles +import wrangles.compute as compute import pandas as pd +import pytest class TestCaseWhen: @@ -349,4 +351,407 @@ def test_search_score(self): assert 'scored_results' in df.columns and 'Score Summary' in df.columns assert isinstance(df.iloc[0]['scored_results'], list) and isinstance(df.iloc[0]['scored_results'][0], dict) assert isinstance(df.iloc[0]['Score Summary'], list) and isinstance(df.iloc[0]['Score Summary'][0], str) - assert len(df.iloc[0]['scored_results']) == 4 and len(df.iloc[0]['Score Summary']) == 4 \ No newline at end of file + assert len(df.iloc[0]['scored_results']) == 4 and len(df.iloc[0]['Score Summary']) == 4 + assert isinstance(df.iloc[0]['scored_results'][0]['part_code_matches'], list) + + def test_part_code_matches_preserve_match_details(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Genuine AB-123 replacement", + "snippet": "Compatible code ZX900", + "link": "https://example.com/products/AB123X", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB-123", "ZX-900"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [ + { + "match_type": "MPN", + "match_level": "exact", + "match_source": "title", + "matched_code": "AB-123", + "input_code": "AB-123", + }, + { + "match_type": "Codes", + "match_level": "stripped", + "match_source": "snippet", + "matched_code": "ZX900", + "input_code": "ZX-900", + }, + ] + + def test_part_code_matches_remove_redundant_less_specific_matches(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "INA NATV6-PP-A track roller", + "snippet": "", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["NATV6-PP-A"], + part_codes=["NATV6-PP-A", "NATV6"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "exact", + "match_source": "title", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6-PP-A", + }] + assert result["part_code_match_count"] == 3 + + def test_part_code_match_inside_larger_code_is_partial(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "INA NATV6-PP-A track roller", + "snippet": "", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + part_codes=["NATV6"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "Codes", + "match_level": "partial", + "match_source": "title", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6", + }] + + def test_part_code_matches_prefer_mpn_over_duplicate_code_evidence(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Genuine AB123 replacement with ZX900", + "snippet": "", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-12"], + part_codes=["AB123", "ZX900"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [ + { + "match_type": "MPN", + "match_level": "partial", + "match_source": "title", + "matched_code": "AB123", + "input_code": "AB-12", + }, + { + "match_type": "Codes", + "match_level": "exact", + "match_source": "title", + "matched_code": "ZX900", + "input_code": "ZX900", + }, + ] + + def test_part_code_matches_url_case_difference_is_exact(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "INA track roller", + "snippet": "", + "link": "https://example.com/products/natv6-pp-a", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["NATV6-PP-A"], + part_codes=["NATV6-PP-A"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "exact", + "match_source": "url", + "matched_code": "natv6-pp-a", + "input_code": "NATV6-PP-A", + }] + assert list(result["part_code_matches"][0]) == [ + "match_type", + "match_level", + "match_source", + "matched_code", + "input_code", + ] + + def test_part_code_match_fallback_reclassifies_identical_token_as_exact(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Product details", + "snippet": "Olympic Part Code: 085-196-225.", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + part_codes=["085-196-225"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "Codes", + "match_level": "exact", + "match_source": "snippet", + "matched_code": "085-196-225", + "input_code": "085-196-225", + }] + + @pytest.mark.parametrize("input_code,source_text,expected_level", [ + ("085-196-225 M10", "085-196-225 M100", "partial"), + ("ABCD", "XYZAB CD1", "partial"), + ("ABC", "ABC.", "exact"), + ("ABC", "LMABC123", None), + ]) + def test_part_code_match_fallback_respects_complete_code_boundaries( + self, input_code, source_text, expected_level + ): + matches = compute._find_part_code_matches( + [input_code], {"Snippet": source_text}, "Codes" + ) + + if expected_level is None: + assert matches == [] + else: + assert len(matches) == 1 + assert matches[0]["match_level"] == expected_level + assert matches[0]["matched_code"] == ( + source_text.rstrip(".") if expected_level == "exact" + else source_text + ) + + def test_part_code_matches_keep_only_one_best_mpn(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "NATV6-PP-A track roller", + "snippet": "Replacement NATV6-PP-A", + "link": "https://example.com/products/natv6-pp-a", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["NATV6-PP-A"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "exact", + "match_source": "title", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6-PP-A", + }] + + def test_sentence_punctuation_does_not_hide_stronger_exact_evidence(self): + matches = compute._find_part_code_matches( + ["AB-123"], {"Snippet": "AB-123. Alternative AB123"}, "Codes" + ) + + assert matches[0]["match_level"] == "exact" + assert matches[0]["matched_code"] == "AB-123" + + def test_part_code_matches_deduplicate_codes_across_sources(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "NATV6-X-PP-A track roller", + "snippet": "Replacement NATV6-X-PP-A", + "link": "https://example.com/products/NATV6-X-PP-A", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + part_codes=["NATV6"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "Codes", + "match_level": "partial", + "match_source": "title", + "matched_code": "NATV6-X-PP-A", + "input_code": "NATV6", + }] + + def test_part_code_matches_remove_codes_contained_within_mpn_match(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Product details", + "snippet": "Part number 085-196-225 M10", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["085-196-225 M10"], + part_codes=["085-196-225 M10", "225 M10", "085-196-225"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "exact", + "match_source": "snippet", + "matched_code": "085-196-225 M10", + "input_code": "085-196-225 M10", + }] + + def test_part_code_matches_remove_code_input_contained_within_mpn(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "LBBR 14-2LS linear ball bearing", + "snippet": "", + "link": ( + "https://example.com/lbbr-14-2ls-hv6-ewellix-" + "stainless-steel-linear-ball-bearing" + ), + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["LBBR14-2LS"], + part_codes=["LBBR14-2LS", "LBBR14"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "stripped", + "match_source": "title", + "matched_code": "LBBR 14-2LS", + "input_code": "LBBR14-2LS", + }] + assert result["part_code_match_count"] == 6 + + def test_part_code_matches_is_empty_when_no_code_matches(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Unrelated product", + "snippet": "No matching identifiers", + "link": "https://example.com/products/unrelated", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB-123"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [] + assert result["part_code_match_count"] == 0 + + def test_part_code_match_count_ignores_duplicate_candidates_and_occurrences(self): + payloads = [{ + "search_results": [{ + "title": "AB-123 replacement for AB-123", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB-123", "AB-123"], + must_match_part_code=False, + )[0] + + assert result["part_code_match_count"] == 2 + assert len(result["part_code_matches"]) == 1 + assert result["part_code_matches"][0]["match_type"] == "MPN" + + def test_part_code_matches_are_nested_in_single_recipe_output(self): + result = wrangles.recipe.run( + """ + wrangles: + - compute.score_search_results: + input: [results, suppliers, part_codes, MPN, Description] + output: scored_results + must_match_part_code: false + """, + dataframe=self.score_data, + ) + + assert "part_code_matches" not in result.columns + assert "part_code_match_count" not in result.columns + assert isinstance(result.iloc[0]["scored_results"][0]["part_code_matches"], list) + assert isinstance(result.iloc[0]["scored_results"][0]["part_code_match_count"], int) + + @pytest.mark.parametrize("must_match_part_code", [True, False]) + def test_part_code_evidence_does_not_change_scoring_or_filtering( + self, monkeypatch, must_match_part_code + ): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [ + { + "title": "Brand AB-123 bearing", + "link": "https://example.com/products/AB-123", + "google_rank": 2, + }, + { + "title": "Brand unrelated bearing", + "link": "https://example.com/products/other", + "google_rank": 1, + }, + ], + }] + kwargs = { + "suppliers": ["Brand"], + "mpns": ["AB-123"], + "part_codes": ["AB-123", "AB12"], + "must_match_part_code": must_match_part_code, + } + with_evidence = compute.score_search_results(payloads, **kwargs) + assert with_evidence[0]["part_code_match_count"] > 0 + + monkeypatch.setattr(compute, "_find_part_code_matches", lambda *args: []) + without_evidence = compute.score_search_results(payloads, **kwargs) + + def without_diagnostics(results): + return [{ + key: value for key, value in result.items() + if key not in {"part_code_matches", "part_code_match_count"} + } for result in results] + + assert without_diagnostics(with_evidence) == without_diagnostics(without_evidence) diff --git a/wrangles/compute.py b/wrangles/compute.py index 73417801..dd178e08 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -1,5 +1,6 @@ -from urllib.parse import urlsplit +import re from typing import List, Tuple, Dict +from urllib.parse import urlsplit # Import our separated helpers from . import web as _web @@ -23,7 +24,7 @@ def _get_supplier_site_score(supplier_names: list, netloc: str) -> int: domain_parts = netloc.split('.') if len(domain_parts) < 2: return 0 - + normalized_domain = _compare.normalize_alphanum(domain_parts[-2]) normalized_suppliers = [_compare.normalize_alphanum(s) for s in supplier_names] @@ -33,13 +34,13 @@ def _get_supplier_site_score(supplier_names: list, netloc: str) -> int: for sup_norm in normalized_suppliers: if len(normalized_domain) > 4 and normalized_domain in sup_norm: return 1 if len(sup_norm) > 4 and sup_norm in normalized_domain: return 1 - + return 0 def _evaluate_part_code_match( candidates: list, - fields_tokens: dict, + fields_tokens: dict, squashed_fields: dict, # <--- NEW PARAMETER exact_score: float, partial_base: float, @@ -54,17 +55,17 @@ def _evaluate_part_code_match( best_score, best_ratio = 0.0, 0.0 best_reason = f"No {entity_name} Match" best_visual = "" - - if not candidates: + + if not candidates: return best_score, best_reason, best_ratio, best_visual - + for candidate in candidates: norm_cand = _compare.normalize_alphanum(candidate) if not norm_cand: continue - + cand_len = len(norm_cand) is_short_code = cand_len < min_length_for_substring - + # --- THE FIX: The Long-Code Bypass --- # If the code is 4+ chars, it's safe to check the squashed text for a perfect embedded match if not is_short_code and squashed_fields: @@ -78,11 +79,11 @@ def _evaluate_part_code_match( for token in tokens: if not token: continue token_len = len(token) - - # 1. Exact Token Match + + # 1. Exact Token Match if norm_cand == token: return exact_score, f"Exact Match '{candidate}' ({entity_name}) in {field_name}", 1.0, f"**{norm_cand}**" - + # 2. Variant / Substring Match (e.g., '555' inside 'LM555CN') elif not is_short_code and norm_cand in token: ratio = cand_len / token_len @@ -96,9 +97,199 @@ def _evaluate_part_code_match( return best_score, best_reason, best_ratio, best_visual +def _find_part_code_matches( + candidates: list, + raw_fields: dict, + match_type: str, + min_length_for_substring: int = 4 +) -> list: + """ + Return each field-level part-code match without reducing the result to a score. + + Match levels are mutually exclusive for a candidate and field: + - exact: the original candidate matches a complete code, ignoring case + - stripped: the candidate matches after removing non-alphanumeric characters + - partial: the normalized candidate is embedded in a larger normalized string + """ + def _find_partial_source_code(source_text: str, norm_cand: str) -> str: + code_tokens = list(re.finditer( + r"[a-z0-9]+(?:[-._][a-z0-9]+)*", + source_text, + re.IGNORECASE + )) + for token_match in code_tokens: + token = token_match.group() + if norm_cand in _compare.normalize_alphanum(token): + return token + + normalized_chars = [] + source_indexes = [] + for index, char in enumerate(source_text): + normalized_char = _compare.normalize_alphanum(char) + if normalized_char: + normalized_chars.extend(normalized_char) + source_indexes.extend([index] * len(normalized_char)) + + match_start = "".join(normalized_chars).find(norm_cand) + if match_start < 0: + return "" + + match_end = match_start + len(norm_cand) - 1 + source_start = source_indexes[match_start] + source_end = source_indexes[match_end] + 1 + # Include the whole source code when the overlap crosses whitespace. + # Otherwise M10 inside M100 would look like an exact match. + for token_match in code_tokens: + if token_match.start() <= source_start < token_match.end(): + source_start = token_match.start() + if token_match.start() < source_end <= token_match.end(): + source_end = token_match.end() + return source_text[source_start:source_end] + + matches = [] + seen = set() + + for candidate in candidates or []: + candidate_text = str(candidate).strip() + norm_cand = _compare.normalize_alphanum(candidate_text) + if not norm_cand: + continue + + # A period is internal to a code only when it connects alphanumerics. + # Sentence punctuation must not hide exact evidence behind a variant. + left_boundary = r"(?= min_length_for_substring: + match_level = "partial" + + if not match_level: + continue + + match = { + "match_type": match_type, + "match_level": match_level, + "match_source": field_name.lower(), + "matched_code": matched_code, + "input_code": candidate_text, + } + match_key = tuple(match.values()) + if match_key not in seen: + seen.add(match_key) + matches.append(match) + + return matches + + +def _reduce_part_code_matches(matches: list) -> list: + """ + Return one best MPN match followed by unique matched Code evidence. + + Duplicate Codes that repeat an MPN input or matched value are omitted. + Repeated Code evidence across fields is reduced to the strongest and most + specific match, preferring title, then snippet, then URL when otherwise tied. + """ + match_level_rank = {"partial": 1, "stripped": 2, "exact": 3} + match_source_rank = {"url": 1, "snippet": 2, "title": 3} + + def _match_rank(match: dict) -> tuple: + return ( + match_level_rank.get(match.get("match_level"), 0), + len(_compare.normalize_alphanum(match.get("input_code", ""))), + match_source_rank.get(match.get("match_source"), 0), + ) + + mpn_matches = [ + match for match in matches + if match.get("match_type") == "MPN" + ] + best_mpn = max(mpn_matches, key=_match_rank, default=None) + normalized_mpn_inputs = { + _compare.normalize_alphanum(match.get("input_code", "")) + for match in mpn_matches + } + best_mpn_matched_code = ( + _compare.normalize_alphanum(best_mpn.get("matched_code", "")) + if best_mpn else "" + ) + best_mpn_input_code = ( + _compare.normalize_alphanum(best_mpn.get("input_code", "")) + if best_mpn else "" + ) + + best_code_matches = {} + code_key_order = [] + + for match in matches: + if match.get("match_type") != "Codes": + continue + + normalized_input = _compare.normalize_alphanum( + match.get("input_code", "") + ) + normalized_matched = _compare.normalize_alphanum( + match.get("matched_code", "") + ) + if normalized_input in normalized_mpn_inputs: + continue + if best_mpn_input_code and normalized_input in best_mpn_input_code: + continue + if ( + best_mpn_matched_code + and normalized_matched in best_mpn_matched_code + ): + continue + + if normalized_matched not in best_code_matches: + code_key_order.append(normalized_matched) + best_code_matches[normalized_matched] = match + elif _match_rank(match) > _match_rank( + best_code_matches[normalized_matched] + ): + best_code_matches[normalized_matched] = match + + reduced_matches = [best_mpn] if best_mpn else [] + reduced_matches.extend( + best_code_matches[key] for key in code_key_order + ) + return reduced_matches + + def _evaluate_match( candidates: List[str], - fields: Dict[str, str], + fields: Dict[str, str], exact_score: float, partial_base: float, entity_name: str, @@ -111,24 +302,24 @@ def _evaluate_match( best_score, best_ratio = 0.0, 0.0 best_reason = f"No {entity_name} Match" best_match_str = "" - - if not candidates or not fields: + + if not candidates or not fields: return best_score, best_reason, best_ratio, best_match_str - + for candidate in candidates: norm_cand = _compare.normalize_alphanum(candidate) if not norm_cand: continue - + for field_name, field_text in fields.items(): if not field_text: continue - + norm_field = _compare.normalize_alphanum(field_text) if not norm_field: continue - + # 1. EXACT MATCH if norm_cand == norm_field: return exact_score, f"Exact Match '{candidate}' ({entity_name}) in {field_name}", 1.0, candidate - + # 2. SUBSTRING BYPASS if norm_cand in norm_field: score = round(1.0 * partial_base, 2) @@ -137,15 +328,15 @@ def _evaluate_match( best_reason = f"Embedded Match '{candidate}' ({entity_name}) in {field_name}" best_match_str = candidate continue - + # 3. TOKENIZED FALLBACKS (Reverse Substring & Fuzzy) clean_field = field_text.replace('/', ' ').replace('-', ' ').replace('.', ' ').replace('_', ' ') tokens = clean_field.split() - + for token in tokens: norm_token = _compare.normalize_alphanum(token) if not norm_token: continue - + # A. Reverse Substring Bypass if len(norm_token) >= 5 and norm_token in norm_cand: score = round(0.95 * partial_base, 2) @@ -163,7 +354,7 @@ def _evaluate_match( best_score, best_ratio = score, ratio best_reason = f"Fuzzy Match '{candidate}' ({entity_name}) in {field_name} [{ratio:.2f}]" best_match_str = candidate - + return best_score, best_reason, best_ratio, best_match_str ### Main Score Function ### @@ -185,7 +376,13 @@ def score_search_results( context_match_base: float = 2.0, fuzzy_match_threshold: float = 0.8 ) -> list: - """Core function that scores a single list of search payloads.""" + """Score search payloads and attach reduced part-code matching evidence. + + Each result includes ``part_code_matches`` and ``part_code_match_count``. + The count covers distinct candidate/type/field matches before reduction, + including the MPN's duplicate in ``part_codes``. It is not an occurrence + count within a field and does not affect scores or filtering. + """ suppliers = suppliers or [] part_codes = part_codes or [] mpns = mpns or [] @@ -198,7 +395,7 @@ def score_search_results( raw_terms.extend(mpns) for d in descriptions: raw_terms.append(d) - raw_terms.extend([w for w in d.split() if len(w) > 2]) + raw_terms.extend([w for w in d.split() if len(w) > 2]) unique_terms = {} for t in raw_terms: @@ -218,7 +415,7 @@ def score_search_results( for item in flat_results: link = item.get("link", "") if not link: continue - + norm_link = _web.normalize_site(link) if norm_link not in deduped_map: deduped_map[norm_link] = item @@ -228,7 +425,7 @@ def score_search_results( unique_raw_results = list(deduped_map.values()) scored_flat_results = [] - + def _get_tokens(text: str) -> list: if not text: return [] clean = text.replace('/', ' ').replace('-', ' ').replace('_', ' ') @@ -245,48 +442,48 @@ def _get_tokens(text: str) -> list: "Snippet": str(item.get("snippet", "")), "URL": str(item.get("link", "")) } - + # 2. Squashed version for the complex Part Code bypass squashed_fields = {k: _compare.normalize_alphanum(v) for k, v in raw_fields.items()} # 3. Create the tokenized version for the short Part Codes & Context Matchers fields_tokens = {k: _get_tokens(v) for k, v in raw_fields.items()} - + # --- REFACTORED CONTEXT MATH --- item_matches = [] # Pool all tokens together from title, snippet, and URL all_tokens = fields_tokens["Title"] + fields_tokens["Snippet"] + fields_tokens["URL"] - + for norm_t, orig_t in unique_terms.items(): term_matched = False - + for token in all_tokens: if not token: continue - + # 1. Exact Token Match if norm_t == token: term_matched = True break - + # 2. Embedded Match (Only if the term is > 3 chars to prevent false positives) elif len(norm_t) > 3 and norm_t in token: term_matched = True break - + # 3. Token Fuzzy Match (Typos) else: ratio, _, _ = _compare.partial_ratio(norm_t, token) if ratio >= fuzzy_match_threshold: term_matched = True break - + if term_matched: item_matches.append(orig_t) context_ratio = len(item_matches) / max(1, len(unique_terms)) context_score = round(context_ratio * context_match_base, 1) - + # Build the specific reason string for context score if item_matches: matched_terms_str = ", ".join([f"'{t}'" for t in item_matches]) @@ -298,7 +495,13 @@ def _get_tokens(text: str) -> list: # Entity Scoring mpn_score, mpn_reason, mpn_ratio, mpn_vis = _evaluate_part_code_match(mpns, fields_tokens, squashed_fields, mpn_exact_score, mpn_partial_base, "MPN") pc_score, pc_reason, pc_ratio, pc_vis = _evaluate_part_code_match(part_codes, fields_tokens, squashed_fields, part_code_exact_score, part_code_partial_base, "Part Code") - + raw_part_code_matches = ( + _find_part_code_matches(mpns, raw_fields, "MPN") + + _find_part_code_matches(part_codes, raw_fields, "Codes") + ) + part_code_match_count = len(raw_part_code_matches) + part_code_matches = _reduce_part_code_matches(raw_part_code_matches) + sup_score, sup_reason, sup_ratio, sup_vis = _evaluate_match(suppliers, raw_fields, supplier_exact_score, supplier_partial_base, "Supplier") if mpn_score >= pc_score: @@ -319,13 +522,13 @@ def _get_tokens(text: str) -> list: part_code_found_enum = "other_code_exact" elif pc_score > 0: part_code_found_enum = "other_code_partial" - + part_code_found = part_code_found_enum != "none" supplier_found = sup_ratio >= fuzzy_match_threshold position = item.get("google_rank", 1) position_weight = _get_position_weight(position) - + supplier_site_score = 0 site = item.get("link", "") try: @@ -360,10 +563,10 @@ def _get_tokens(text: str) -> list: "score": total_score, "part_match_score": best_pc_score, "part_match_reason": pc_match_reason, - "part_match_visual": best_vis, + "part_match_visual": best_vis, "brand_score": sup_score, "brand_match_reason": sup_reason, - "brand_match_visual": sup_vis, + "brand_match_visual": sup_vis, } for k, val in sorted_remainder.items(): scoring_details[k] = val @@ -377,10 +580,10 @@ def _get_tokens(text: str) -> list: reason = "no part code match" summary = { - "scored_result_index": 0, + "scored_result_index": 0, "input_row_id": item.get("input_row_id"), "source": item.get("source"), - "score": total_score, + "score": total_score, "link": item.get("link"), "part_code_found": part_code_found_enum, # Storing the enum string here momentarily "brand_found": brand_found, @@ -417,18 +620,20 @@ def _get_tokens(text: str) -> list: scored_item = { "summary": summary, + "part_code_matches": part_code_matches, + "part_code_match_count": part_code_match_count, "pricing": formatted_pricing, "scoring_details": scoring_details, "metadata": metadata } - + scored_flat_results.append(scored_item) filtered_in = sorted([s for s in scored_flat_results if not s["summary"]["filtered"]], key=lambda x: x["summary"]["score"], reverse=True) filtered_out = sorted([s for s in scored_flat_results if s["summary"]["filtered"]], key=lambda x: x["summary"]["score"], reverse=True) combined_results = filtered_in + filtered_out - + for idx, res in enumerate(combined_results, start=1): res["summary"]["scored_result_index"] = idx - return combined_results \ No newline at end of file + return combined_results diff --git a/wrangles/recipe_wrangles/compute.py b/wrangles/recipe_wrangles/compute.py index 5f1c62e0..9ad5aa07 100644 --- a/wrangles/recipe_wrangles/compute.py +++ b/wrangles/recipe_wrangles/compute.py @@ -83,7 +83,7 @@ def score_search_results( ) -> _pd.DataFrame: """ type: object - description: Scores and filters search results based on progressive partial/exact matching. Can return dictionaries or a parallel list of formatted strings. + description: Scores and filters search results based on progressive partial/exact matching. Each result includes reduced part_code_matches evidence and part_code_match_count before reduction. Can return dictionaries or a parallel list of formatted strings. additionalProperties: false required: - input