Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public class OpenSearchIndexBootStrapper extends Thread {
private static final String CONFIG_REPLICAS = CONFIG_PREFIX + ".no.replica";
private static final String CONFIG_MAX_RETRY = CONFIG_PREFIX + ".max.retry";
private static final String SCHEMA_FILE_NAME = "ranger_opensearch_schema.json";
private static final String KEY_LOWER_CASE_NORMALIZER = "key_lower_case";

private volatile RestClient client;
private String index;
Expand Down Expand Up @@ -176,8 +177,12 @@ private void createIndex() throws IOException {
}

private String buildCreateIndexBody(String mappingsJson) {
// Same role as Solr key_lower_case: one token, lowercased.
String normalizer = "\"" + KEY_LOWER_CASE_NORMALIZER + "\":{\"type\":\"custom\",\"filter\":[\"lowercase\"]}";

return "{\"settings\":{\"number_of_shards\":" + noOfShards
+ ",\"number_of_replicas\":" + noOfReplicas
+ ",\"analysis\":{\"normalizer\":{" + normalizer + "}}"
+ "},\"mappings\":" + mappingsJson + "}";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,40 @@
"index": false
},
"access": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"action": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"agent": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"agentHost": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"cliIP": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"cliType": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"cluster": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"reqContext": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"enforcer": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"event_count": {
"type": "long",
Expand All @@ -58,42 +67,50 @@
"store": true
},
"logType": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"policy": {
"type": "long",
"doc_values": true
},
"proxyUsers": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"reason": {
"type": "text"
},
"repo": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"repoType": {
"type": "integer",
"doc_values": true
},
"req_caller_id": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"req_self_id": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"reqData": {
"type": "text"
},
"reqUser": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"resType": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"resource": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"result": {
"type": "integer"
Expand All @@ -103,19 +120,23 @@
"doc_values": true
},
"sess": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"tags": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"tags_str": {
"type": "text"
},
"datasets": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"projects": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"datasetIds": {
"type": "long"
Expand All @@ -124,7 +145,8 @@
"type": "text"
},
"zoneName": {
"type": "keyword"
"type": "keyword",
"normalizer": "key_lower_case"
},
"policyVersion": {
"type": "long"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,17 @@ private Map<String, Object> buildClause(String fieldName, SearchField.DATA_TYPE
String value = paramValue.toString().trim();

if (!value.isEmpty()) {
// Exact string matches on keyword fields use a case-insensitive term query of the original value.
// reason stays a match phrase because that field is mapped as text.
if (dataType == SearchField.DATA_TYPE.STRING && !"reason".equals(fieldName)) {
Map<String, Object> termBody = new LinkedHashMap<>();

termBody.put("value", value);
termBody.put("case_insensitive", true);

return Map.of("term", Map.of(fieldName, termBody));
}

return Map.of("match_phrase", Map.of(fieldName, escapeLucene(value.toLowerCase())));
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,20 +101,44 @@ void buildSearchBody_partialStringSearch() throws Exception {
}

@Test
void buildSearchBody_fullStringMatch() throws Exception {
void buildSearchBody_fullStringMatchKeepsOriginalValueAndIgnoresCase() throws Exception {
SearchCriteria criteria = new SearchCriteria();

criteria.setMaxRows(10);
criteria.setStartIndex(0);
criteria.addParam("accessType", "read");
criteria.addParam("accessType", "SELECT");
criteria.addParam("action", "METADATA OPERATION");

List<SearchField> searchFields = List.of(new SearchField("accessType", "access", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
List<SearchField> searchFields = List.of(
new SearchField("accessType", "access", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL),
new SearchField("action", "action", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
List<SortField> sortFields = List.of(new SortField("eventTime", "evtTime", true, SortField.SORT_ORDER.DESC));
String body = openSearchUtil.buildSearchBody(criteria, searchFields, sortFields);
JsonNode must = MAPPER.readTree(body).at("/query/bool/must");
JsonNode accessTerm = mustClause(must, "term", "access");
JsonNode actionTerm = mustClause(must, "term", "action");

assertEquals("SELECT", accessTerm.get("value").asText());
assertTrue(accessTerm.get("case_insensitive").asBoolean());
assertEquals("METADATA OPERATION", actionTerm.get("value").asText());
assertTrue(actionTerm.get("case_insensitive").asBoolean());
}

@Test
void buildSearchBody_reasonStaysMatchPhrase() throws Exception {
SearchCriteria criteria = new SearchCriteria();

criteria.setMaxRows(10);
criteria.setStartIndex(0);
criteria.addParam("reason", "Denied");

List<SearchField> searchFields = List.of(new SearchField("reason", "reason", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
List<SortField> sortFields = List.of(new SortField("eventTime", "evtTime", true, SortField.SORT_ORDER.DESC));
String body = openSearchUtil.buildSearchBody(criteria, searchFields, sortFields);
JsonNode must = MAPPER.readTree(body).at("/query/bool/must");
JsonNode reason = mustClause(must, "match_phrase", "reason");

assertTrue(body.contains("match_phrase"));
assertTrue(body.contains("access"));
assertTrue(body.contains("read"));
assertEquals("denied", reason.asText());
}

@Test
Expand Down Expand Up @@ -216,4 +240,21 @@ void escapeLucene_noSpecialChars() {

assertEquals("simplevalue", escaped);
}

private static JsonNode mustClause(JsonNode must, String queryType, String fieldName) {
JsonNode result = null;

for (JsonNode clause : must) {
JsonNode typed = clause.get(queryType);

if (typed != null && typed.has(fieldName)) {
result = typed.get(fieldName);
break;
}
}

assertNotNull(result);

return result;
}
}