From 1fef3f0d68a15b0177e5598db8559787592fe667 Mon Sep 17 00:00:00 2001 From: Saksham Date: Thu, 12 Nov 2026 15:16:47 +0100 Subject: [PATCH] feat(search): boost default fields and require every word * configure the upstream QueryParser with an explicit, boosted field list that also covers DOIs, resource types, communities and CERN custom fields * add CDSSearchFieldTransformer: words separated by spaces must all match, each in any field (an explicit AND; OpenSearch's default_operator is per field and finds nothing for "thesis Mobius") * search words such as report numbers and DOIs as phrases, so they can be pasted without quotes (a bare "/" fails in query_string) * move the search aliases to CDS_SEARCH_ALIASES and add shortcuts for contributors, affiliations, subjects, resource type, communities and more CERN fields * keep the upstream restricted terms (internal_notes, _exists_) that the previous alias mapping dropped * use the same parser for drafts and community records search Co-Authored-By: Claude Opus 5.5 --- invenio.cfg | 159 +++++++++++++++++--------- site/cds_rdm/search/__init__.py | 12 ++ site/cds_rdm/search/queryparser.py | 128 +++++++++++++++++++++ site/tests/test_search_queryparser.py | 104 +++++++++++++++++ 4 files changed, 347 insertions(+), 56 deletions(-) create mode 100644 site/cds_rdm/search/__init__.py create mode 100644 site/cds_rdm/search/queryparser.py create mode 100644 site/tests/test_search_queryparser.py diff --git a/invenio.cfg b/invenio.cfg index 2f61b483..60a9336d 100644 --- a/invenio.cfg +++ b/invenio.cfg @@ -49,10 +49,7 @@ from invenio_rdm_records.checks import requests as checks_requests from invenio_rdm_records.config import (always_valid, RDM_RECORDS_PERSONORG_SCHEMES, RDM_RECORDS_IDENTIFIERS_SCHEMES as RDM_RECORDS_RELATED_IDENTIFIERS_SCHEMES ) -from invenio_records_resources.services.records.queryparser import ( - QueryParser, - SearchFieldTransformer, -) +from invenio_records_resources.services.records.queryparser import QueryParser from invenio_rdm_records.proxies import current_rdm_records_service as record_service from invenio_rdm_records.services.components import DefaultRecordsComponents from invenio_rdm_records.services.components.vcs import VCSComponent @@ -60,8 +57,10 @@ from invenio_rdm_records.services.vcs.release import RDMVCSRelease from invenio_rdm_records.config import ( RDM_PERSISTENT_IDENTIFIERS, RDM_PARENT_PERSISTENT_IDENTIFIERS, - RDM_SEARCH + RDM_SEARCH, + RDM_SEARCH_DRAFTS, ) +from cds_rdm.search import DEFAULT_SEARCH_FIELDS, CDSSearchFieldTransformer from invenio_rdm_records.notifications.vcs import ( RepositoryReleaseCommunityRequiredNotificationBuilder, RepositoryReleaseCommunitySubmittedNotificationBuilder, @@ -212,7 +211,6 @@ APP_RDM_ROUTES["index"] = ("/", frontpage_view_function) COMMUNITIES_PERMISSION_POLICY = CDSCommunitiesPermissionPolicy COMMUNITIES_ADMINISTRATION_DISABLED = False COMMUNITIES_ALLOW_RESTRICTED = True -COMMUNITIES_RECORDS_SEARCH = deepcopy(RDM_SEARCH) COMMUNITIES_COLLECTIONS_ENABLED = True # Invenio-Records-Resources @@ -638,59 +636,108 @@ CDS_CERN_SCIENTIFIC_RESOURCE_TYPES = { Resource types that should be included in the CERN Scientific community. """ +CDS_SEARCH_ALIASES = { + "accelerator": "custom_fields.cern\\:accelerators.title.en", + "accelerator.id": "custom_fields.cern\\:accelerators.id", + "accelerator.title": "custom_fields.cern\\:accelerators.title.en", + "accelerators": "custom_fields.cern\\:accelerators.title.en", + "accelerators.id": "custom_fields.cern\\:accelerators.id", + "accelerators.title": "custom_fields.cern\\:accelerators.title.en", + "experiment": "custom_fields.cern\\:experiments.title.en", + "experiment.id": "custom_fields.cern\\:experiments.id", + "experiment.title": "custom_fields.cern\\:experiments.title.en", + "experiments": "custom_fields.cern\\:experiments.title.en", + "experiments.id": "custom_fields.cern\\:experiments.id", + "experiments.title": "custom_fields.cern\\:experiments.title.en", + "department": "custom_fields.cern\\:departments.title.en", + "department.id": "custom_fields.cern\\:departments.id", + "department.title": "custom_fields.cern\\:departments.title.en", + "departments": "custom_fields.cern\\:departments.title.en", + "departments.id": "custom_fields.cern\\:departments.id", + "departments.title": "custom_fields.cern\\:departments.title.en", + "programme": "custom_fields.cern\\:programmes.title.en", + "programme.id": "custom_fields.cern\\:programmes.id", + "programme.title": "custom_fields.cern\\:programmes.title.en", + "programmes": "custom_fields.cern\\:programmes.title.en", + "programmes.id": "custom_fields.cern\\:programmes.id", + "programmes.title": "custom_fields.cern\\:programmes.title.en", + "project": "custom_fields.cern\\:projects", + "projects": "custom_fields.cern\\:projects", + "study": "custom_fields.cern\\:studies", + "studies": "custom_fields.cern\\:studies", + "identifier": "metadata.identifiers.identifier", + "cdsrn": "metadata.identifiers.identifier", + "report_number": "metadata.identifiers.identifier", + "inspire": "metadata.related_identifiers.identifier", + "inis": "metadata.related_identifiers.identifier", + "indico": "metadata.related_identifiers.identifier", + "cds": "metadata.identifiers.identifier", + "aleph": "metadata.identifiers.identifier", + "doi": "pids.doi.identifier", + "language": "metadata.languages.id", + "languages": "metadata.languages.id", + "title": "metadata.title", + "publisher": "metadata.publisher", + "description": "metadata.description", + "publication_date": "metadata.publication_date", + "creator": "metadata.creators.person_or_org.name", + "author": "metadata.creators.person_or_org.name", + "creators": "metadata.creators.person_or_org.name", + "authors": "metadata.creators.person_or_org.name", + "contributor": "metadata.contributors.person_or_org.name", + "contributors": "metadata.contributors.person_or_org.name", + "affiliation": "metadata.creators.affiliations.name", + "affiliations": "metadata.creators.affiliations.name", + "subject": "metadata.subjects.subject", + "subjects": "metadata.subjects.subject", + "type": "metadata.resource_type.title.en", + "resource_type": "metadata.resource_type.title.en", + "resource_type.id": "metadata.resource_type.id", + "community": "parent.communities.entries.metadata.title", + "communities": "parent.communities.entries.metadata.title", + "community.id": "parent.communities.entries.slug", + "communities.id": "parent.communities.entries.slug", + "apprn": "metadata.identifiers.identifier", + "edms": "metadata.related_identifiers.identifier", + "hal": "metadata.related_identifiers.identifier", + "archive": "metadata.related_identifiers.identifier", + "beam": "custom_fields.cern\\:beams.title.en", + "beams": "custom_fields.cern\\:beams.title.en", + "committee": "custom_fields.cern\\:committees.title.en", + "committees": "custom_fields.cern\\:committees.title.en", + "facility": "custom_fields.cern\\:facilities", + "facilities": "custom_fields.cern\\:facilities", + "meeting": "custom_fields.meeting\\:meeting.title", +} +"""Search shortcuts (``alias:value``) available in the record search bar. + +Documented for users in docs-cds-rdm, docs/search/search.md: keep both in sync. +""" + +_CDS_SEARCH_PARSER = QueryParser.factory( + fields=DEFAULT_SEARCH_FIELDS, + # Skip fields a word can't be parsed for (e.g. dates) instead of failing. + lenient=True, + # Keep the upstream restricted terms (internal_notes.*, _exists_), which only + # system users may search, and add the CDS aliases on top. + mapping={ + **RDM_SEARCH["query_parser_cls"].keywords["extra_params"]["mapping"], + **CDS_SEARCH_ALIASES, + }, + tree_transformer_cls=CDSSearchFieldTransformer, +) + RDM_SEARCH = { **deepcopy(RDM_SEARCH), - "query_parser_cls": QueryParser.factory( - mapping={ - "accelerator": "custom_fields.cern\:accelerators.title.en", - "accelerator.id": "custom_fields.cern\:accelerators.id", - "accelerator.title": "custom_fields.cern\:accelerators.title.en", - "accelerators": "custom_fields.cern\:accelerators.title.en", - "accelerators.id": "custom_fields.cern\:accelerators.id", - "accelerators.title": "custom_fields.cern\:accelerators.title.en", - "experiment": "custom_fields.cern\:experiments.title.en", - "experiment.id": "custom_fields.cern\:experiments.id", - "experiment.title": "custom_fields.cern\:experiments.title.en", - "experiments": "custom_fields.cern\:experiments.title.en", - "experiments.id": "custom_fields.cern\:experiments.id", - "experiments.title": "custom_fields.cern\:experiments.title.en", - "department": "custom_fields.cern\:departments.title.en", - "department.id": "custom_fields.cern\:departments.id", - "department.title": "custom_fields.cern\:departments.title.en", - "departments": "custom_fields.cern\:departments.title.en", - "departments.id": "custom_fields.cern\:departments.id", - "departments.title": "custom_fields.cern\:departments.title.en", - "programme": "custom_fields.cern\:programmes.title.en", - "programme.id": "custom_fields.cern\:programmes.id", - "programme.title": "custom_fields.cern\:programmes.title.en", - "programmes": "custom_fields.cern\:programmes.title.en", - "programmes.id": "custom_fields.cern\:programmes.id", - "programmes.title": "custom_fields.cern\:programmes.title.en", - "project": "custom_fields.cern\:projects", - "projects": "custom_fields.cern\:projects", - "study": "custom_fields.cern\:studies", - "studies": "custom_fields.cern\:studies", - "identifier": "metadata.identifiers.identifier", - "cdsrn": "metadata.identifiers.identifier", - "report_number": "metadata.identifiers.identifier", - "inspire": "metadata.related_identifiers.identifier", - "inis": "metadata.related_identifiers.identifier", - "indico": "metadata.related_identifiers.identifier", - "cds": "metadata.identifiers.identifier", - "aleph": "metadata.identifiers.identifier", - "doi": "pids.doi.identifier", - "language": "metadata.languages.id", - "languages": "metadata.languages.id", - "title": "metadata.title", - "publisher": "metadata.publisher", - "description": "metadata.description", - "publication_date": "metadata.publication_date", - "creator": "metadata.creators.person_or_org.name", - "creators": "metadata.creators.person_or_org.name", + "query_parser_cls": _CDS_SEARCH_PARSER, +} - }, - tree_transformer_cls=SearchFieldTransformer, - ), +# Defined after RDM_SEARCH so community record pages search the same way. +COMMUNITIES_RECORDS_SEARCH = deepcopy(RDM_SEARCH) + +RDM_SEARCH_DRAFTS = { + **deepcopy(RDM_SEARCH_DRAFTS), + "query_parser_cls": _CDS_SEARCH_PARSER, } RDM_PERMISSION_POLICY = CDSRDMRecordPermissionPolicy diff --git a/site/cds_rdm/search/__init__.py b/site/cds_rdm/search/__init__.py new file mode 100644 index 00000000..5614da17 --- /dev/null +++ b/site/cds_rdm/search/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the GPL-2.0 License; see LICENSE file for more details. + +"""CDS-RDM record search customisations.""" + +from .queryparser import DEFAULT_SEARCH_FIELDS, CDSSearchFieldTransformer + +__all__ = ("CDSSearchFieldTransformer", "DEFAULT_SEARCH_FIELDS") diff --git a/site/cds_rdm/search/queryparser.py b/site/cds_rdm/search/queryparser.py new file mode 100644 index 00000000..7e3ddddb --- /dev/null +++ b/site/cds_rdm/search/queryparser.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the GPL-2.0 License; see LICENSE file for more details. + +"""Search configuration for the CDS record search bar. + +Used with the upstream ``QueryParser`` (see ``invenio.cfg``):: + + QueryParser.factory( + fields=DEFAULT_SEARCH_FIELDS, + lenient=True, + mapping={...}, + tree_transformer_cls=CDSSearchFieldTransformer, + ) +""" + +import re + +from invenio_records_resources.services.records.queryparser import ( + SearchFieldTransformer, +) +from luqum.tree import ( + AndOperation, + Group, + OrOperation, + Phrase, + Plus, + Prohibit, + Word, +) + +DEFAULT_SEARCH_FIELDS = [ + # Mirrors ``index.query.default_field`` of the upstream record mapping + # (invenio-rdm-records, rdmrecords/records/record-v7.0.0.json), with boosts. + "id^10", + "metadata.title^10", + "metadata.title.original^10", + "metadata.additional_titles.title^5", + "metadata.creators.person_or_org.name^5", + "metadata.creators.person_or_org.family_name^5", + "metadata.creators.person_or_org.given_name^2", + "metadata.creators.affiliations.name", + "metadata.contributors.person_or_org.name^2", + "metadata.contributors.person_or_org.family_name^2", + "metadata.contributors.person_or_org.given_name", + "metadata.contributors.affiliations.name", + "metadata.identifiers.identifier^10", + "metadata.subjects.subject^3", + "metadata.description", + "metadata.additional_descriptions.description", + "metadata.contact", + "metadata.formats", + "metadata.funding.award.identifiers.identifier", + "metadata.funding.award.acronym.text", + "metadata.funding.award.number", + "metadata.funding.funder.name", + "metadata.locations.features.place", + "metadata.locations.features.description", + "metadata.publication_date", + "metadata.publisher", + "metadata.version", + "metadata.dates.description", + "metadata.references.reference", + # Not in the upstream default fields. + "pids.doi.identifier^10", + "metadata.related_identifiers.identifier^3", + "metadata.resource_type.title.en^3", + "parent.communities.entries.metadata.title^2", + "custom_fields.cern:experiments.title.en^2", + "custom_fields.cern:accelerators.title.en^2", + "custom_fields.cern:departments.title.en", + "custom_fields.cern:programmes.title.en", + "custom_fields.cern:beams.title.en", + "custom_fields.cern:committees.title.en", + "custom_fields.cern:projects", + "custom_fields.cern:studies", + "custom_fields.cern:facilities", + "custom_fields.meeting:meeting.title", + "custom_fields.journal:journal.title", + "custom_fields.thesis:thesis.university", +] +"""Fields searched by a query without field qualifiers, with boosts. + +Never use wildcards here: ``*`` would also search fields restricted to +administrators, such as ``internal_notes``. +""" + +# A word the analyzer splits into several tokens: report numbers, DOIs, +# hyphenated words. Not words with wildcards or escapes. +_COMPOUND_WORD_RE = re.compile(r"^[^*?\\\"]+[-/][^*?\\\"]+$") + + +class CDSSearchFieldTransformer(SearchFieldTransformer): + """Field aliases (upstream), plus: every word must match. + + OpenSearch's ``default_operator: AND`` is applied per field when the fields + use different analyzers: ``thesis Mobius`` would need both words in the + same field and match nothing. An explicit ``AND`` lets each word match in + any field. + """ + + def visit_unknown_operation(self, node, context): + """Turn ``a b`` into ``a AND b``, unless ``+``/``-`` say what is required.""" + children = list(self.clone_children(node, node.clone_item(), context)) + if any(isinstance(child, (Plus, Prohibit)) for child in children): + new_node = node.clone_item() + new_node.children = children + yield new_node + return + # "a b OR c" is "a (b OR c)": keep the group, OpenSearch would read + # "a AND b OR c" as "+a +b c". + yield AndOperation( + *(Group(c) if isinstance(c, OrOperation) else c for c in children) + ) + + def visit_word(self, node, context): + """Search a word made of several tokens as a phrase. + + ``CERN-THESIS-2021-078`` must match as a whole, not any of its parts, + and the ``/`` of a DOI would otherwise start a regular expression. + """ + for new_node in super().visit_word(node, context): + if isinstance(new_node, Word) and _COMPOUND_WORD_RE.match(new_node.value): + new_node = Phrase(f'"{new_node.value}"') + yield new_node diff --git a/site/tests/test_search_queryparser.py b/site/tests/test_search_queryparser.py new file mode 100644 index 00000000..104407d7 --- /dev/null +++ b/site/tests/test_search_queryparser.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the GPL-2.0 License; see LICENSE file for more details. + +"""Tests for the CDS record search query parser configuration.""" + +import pytest +from flask_principal import AnonymousIdentity +from invenio_access.permissions import system_identity, system_permission +from invenio_records_resources.services.records.queryparser import QueryParser +from invenio_records_resources.services.records.queryparser.transformer import ( + RestrictedTerm, +) + +from cds_rdm.search import DEFAULT_SEARCH_FIELDS, CDSSearchFieldTransformer + +MAPPING = { + "doi": "pids.doi.identifier", + "creator": "metadata.creators.person_or_org.name", + "internal_notes.note": RestrictedTerm(system_permission), +} + + +def parse(query, identity=None): + """Parse ``query`` with the parser configured as in ``invenio.cfg``.""" + parser_cls = QueryParser.factory( + fields=DEFAULT_SEARCH_FIELDS, + lenient=True, + mapping=MAPPING, + tree_transformer_cls=CDSSearchFieldTransformer, + ) + return parser_cls(identity).parse(query).to_dict() + + +def sent(query): + """The query string sent to OpenSearch, whitespace normalised.""" + return " ".join(parse(query)["query_string"]["query"].split()) + + +def test_plain_query_searches_the_default_fields(): + assert parse("higgs") == { + "query_string": { + "query": "higgs", + "fields": DEFAULT_SEARCH_FIELDS, + "lenient": True, + } + } + + +@pytest.mark.parametrize( + "query,expected", + [ + ("thesis Mobius", "thesis AND Mobius"), + ( + "creator:mobius thesis", + "metadata.creators.person_or_org.name:mobius AND thesis", + ), + ("a b OR c", "a AND (b OR c)"), + ('"pixel detector" atlas', '"pixel detector" AND atlas'), + ("creator:(john doe)", "metadata.creators.person_or_org.name:(john AND doe)"), + # Explicit operators are kept. + ("higgs OR boson", "higgs OR boson"), + ("higgs AND boson", "higgs AND boson"), + # +/- already say what is required: left alone. + ("+thesis mobius^5", "+thesis mobius^5"), + ("higgs -boson", "higgs -boson"), + ], +) +def test_space_means_and(query, expected): + assert sent(query) == expected + + +@pytest.mark.parametrize( + "query,expected", + [ + ("10.17181/CERN.ABCD.1234", '"10.17181/CERN.ABCD.1234"'), + ("doi:10.1007/x", 'pids.doi.identifier:"10.1007/x"'), + ("CERN-THESIS-2021-078", '"CERN-THESIS-2021-078"'), + ("state-of-the-art detector", '"state-of-the-art" AND detector'), + # Left alone: wildcards, escapes, regular expressions, a leading "-". + ("CERN-THESIS-*", "CERN-THESIS-*"), + (r"10.1007\/x", r"10.1007\/x"), + ("/high.*energy/", "/high.*energy/"), + ("higgs -boson", "higgs -boson"), + ], +) +def test_compound_words_are_phrases(query, expected): + assert sent(query) == expected + + +def test_default_fields_have_no_wildcards(): + # "*" would also search restricted fields such as internal_notes. + assert not any("*" in field for field in DEFAULT_SEARCH_FIELDS) + + +def test_restricted_terms_still_restricted(app): + # A normal user's search on a restricted field is rejected and falls back to a + # plain multi_match on the text, so it never queries internal_notes. + assert "multi_match" in parse("internal_notes.note:secret", AnonymousIdentity()) + query = parse("internal_notes.note:secret", system_identity) + assert query["query_string"]["query"] == "internal_notes.note:secret"