diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBaseService.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBaseService.java index 0e26a2a2cb..1c4610f41e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBaseService.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBaseService.java @@ -35,6 +35,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.ranger.plugin.util.JsonUtilsV2; import org.apache.ranger.plugin.util.ServiceDefUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,6 +65,9 @@ public abstract class RangerBaseService { private static final String PROP_DEFAULT_POLICY_NAME_SUFFIX = "name"; private static final String PROP_RESOURCE_FLAG_SUFFIX_IS_EXCLUDES = ".is-excludes"; private static final String PROP_RESOURCE_FLAG_SUFFIX_IS_RECURSIVE = ".is-recursive"; + private static final String PROP_DEFAULT_FULL_POLICY = "default.policy."; + private static final String PROP_DEFAULT_POLICY_USERS = "default.policy.users"; + private static final String PROP_DEFAULT_POLICY_GROUPS = "default.policy.groups"; protected final RangerAdminConfig config; protected RangerServiceDef serviceDef; @@ -164,15 +168,22 @@ public List getDefaultRangerPolicies() throws Exception { if (additionalDefaultPolicySetup) { LOG.info("{}: looking for additional default policies in service-config", getServiceName()); - Set policyIndexes = new TreeSet<>(); + Set policyIndexes = new TreeSet<>(); + List fullPolicyConfigNames = new ArrayList<>(); for (String configName : configs.keySet()) { if (configName.startsWith(PROP_DEFAULT_POLICY_PREFIX) && configName.endsWith(PROP_DEFAULT_POLICY_NAME_SUFFIX)) { policyIndexes.add(configName.substring(PROP_DEFAULT_POLICY_PREFIX.length(), configName.length() - PROP_DEFAULT_POLICY_NAME_SUFFIX.length() - 1)); + } else if (isDefaultFullPolicyConfig(configName)) { + fullPolicyConfigNames.add(configName); } } - LOG.info("{}: found {} additional default policies in service-config", getServiceName(), policyIndexes.size()); + LOG.info("{}: found {} additional default policies in service-config: {} in '{}.*' format, {} in '{}' JSON format", getServiceName(), policyIndexes.size() + fullPolicyConfigNames.size(), policyIndexes.size(), PROP_DEFAULT_POLICY_PREFIX, fullPolicyConfigNames.size(), PROP_DEFAULT_FULL_POLICY); + + for (String configName : fullPolicyConfigNames) { + addDefaultPolicyFromJson(ret, configName, configs.get(configName)); + } for (String policyIndex : policyIndexes) { String policyPropertyPrefix = PROP_DEFAULT_POLICY_PREFIX + policyIndex + "."; @@ -368,6 +379,45 @@ private void addCustomRangerDefaultPolicies(List ret, Map PROP_DEFAULT_FULL_POLICY.length() && !configName.equals(PROP_DEFAULT_POLICY_USERS) && !configName.equals(PROP_DEFAULT_POLICY_GROUPS); + } + + private void addDefaultPolicyFromJson(List ret, String configName, String policyJson) { + RangerPolicy policy = null; + + if (StringUtils.isNotBlank(policyJson)) { + try { + policy = JsonUtilsV2.jsonToObj(policyJson, RangerPolicy.class); + } catch (IOException e) { + LOG.error("{}: failed to parse '{}' in service-config as a policy. Ignored", getServiceName(), configName, e); + } + } + + if (policy == null) { + LOG.warn("{}: no policy specified in '{}'. Ignored", getServiceName(), configName); + } else if (MapUtils.isEmpty(policy.getResources())) { + LOG.warn("{}: no resources specified for default policy '{}' in '{}'. Ignored", getServiceName(), policy.getName(), configName); + } else { + if (StringUtils.isBlank(policy.getName())) { + policy.setName(configName); + } + + policy.setId(null); + policy.setGuid(null); + policy.setVersion(1L); + policy.setService(serviceName); + + if (StringUtils.isEmpty(policy.getDescription())) { + policy.setDescription(configName); + } + + LOG.info("{}: adding default policy from '{}': name={}", getServiceName(), configName, policy.getName()); + + ret.add(policy); + } + } + private RangerPolicy getDefaultPolicy(List resourceHierarchy) { LOG.debug("==> RangerBaseService.getDefaultPolicy()"); @@ -449,7 +499,7 @@ private List getUserList() { uniqueUsers.add(serviceConfig.get("username")); } - String defaultUsers = serviceConfig.get("default.policy.users"); + String defaultUsers = serviceConfig.get(PROP_DEFAULT_POLICY_USERS); if (!StringUtils.isEmpty(defaultUsers)) { String[] defaultUserList = StringUtils.split(defaultUsers, ","); @@ -474,7 +524,7 @@ private List getGroupList() { Map serviceConfig = service.getConfigs(); if (serviceConfig != null) { - String defaultGroups = serviceConfig.get("default.policy.groups"); + String defaultGroups = serviceConfig.get(PROP_DEFAULT_POLICY_GROUPS); if (!StringUtils.isEmpty(defaultGroups)) { List defaultGroupList = new ArrayList<>(Arrays.asList(StringUtils.split(defaultGroups, ","))); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBaseService.java b/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBaseService.java index 9dd1255f80..37ed70f10c 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBaseService.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBaseService.java @@ -34,16 +34,19 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -262,4 +265,220 @@ public void test8_GetLookupUser_SimpleAuthReturnsNull() throws IOException { String user = svc.getLookupUser("simple", null, null); assertEquals(null, user); } + + @Test + public void test9_DefaultPolicyFromJson() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", "{\"id\": 42, \"service\": \"other\", \"name\": \"p-json\", \"isDenyAllElse\": true," + + " \"resources\": {\"database\": {\"values\": [\"db1\", \"db2\"], \"isExcludes\": true}, \"table\": {\"values\": [\"t1\"]}}," + + " \"policyItems\": [{\"users\": [\"u1\"], \"groups\": [\"g1\"], \"roles\": [\"r1\"], \"accesses\": [{\"type\": \"select\"}], \"delegateAdmin\": true}]," + + " \"denyPolicyItems\": [{\"users\": [\"u2\"], \"accesses\": [{\"type\": \"update\"}]}]}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(1, policies.size()); + RangerPolicy p = policies.get(0); + assertEquals("p-json", p.getName()); + assertEquals("svc1", p.getService()); + assertNull(p.getId()); + assertEquals(1L, p.getVersion()); + assertEquals("default.policy.1", p.getDescription()); + assertTrue(p.getIsEnabled()); + assertTrue(p.getIsAuditEnabled()); + assertTrue(p.getIsDenyAllElse()); + assertEquals(Arrays.asList("db1", "db2"), p.getResources().get("database").getValues()); + assertTrue(p.getResources().get("database").getIsExcludes()); + assertEquals(Collections.singletonList("t1"), p.getResources().get("table").getValues()); + assertEquals(1, p.getPolicyItems().size()); + RangerPolicy.RangerPolicyItem item = p.getPolicyItems().get(0); + assertEquals(Collections.singletonList("u1"), item.getUsers()); + assertEquals(Collections.singletonList("g1"), item.getGroups()); + assertEquals(Collections.singletonList("r1"), item.getRoles()); + assertTrue(item.getDelegateAdmin()); + assertEquals("select", item.getAccesses().get(0).getType()); + assertTrue(item.getAccesses().get(0).getIsAllowed()); + assertEquals(Collections.singletonList("u2"), p.getDenyPolicyItems().get(0).getUsers()); + } + + @Test + public void test10_DefaultPolicyFromJson_CombinedWithDefaultPolicyConfigs() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default-policy.1.name", "p-props"); + cfg.put("default-policy.1.resource.database", "db1"); + cfg.put("default-policy.1.policyItem.1.users", "u1"); + cfg.put("default-policy.1.policyItem.1.accessTypes", "select"); + cfg.put("default.policy.1", "{\"name\": \"p-json-1\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + cfg.put("default.policy.2", "{\"name\": \"p-json-2\", \"resources\": {\"database\": {\"values\": [\"db2\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(3, policies.size()); + assertTrue(policies.stream().anyMatch(p -> "p-props".equals(p.getName()))); + assertTrue(policies.stream().anyMatch(p -> "p-json-1".equals(p.getName()))); + assertTrue(policies.stream().anyMatch(p -> "p-json-2".equals(p.getName()))); + } + + @Test + public void test11_DefaultPolicyFromJson_InvalidJsonIgnored() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", "{not-a-json"); + cfg.put("default.policy.2", "{\"name\": \"valid\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(1, policies.size()); + assertEquals("valid", policies.get(0).getName()); + } + + @Test + public void test12_DefaultPolicyFromJson_SkipsPoliciesWithoutResources() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", "{\"resources\": {}}"); + cfg.put("default.policy.2", "{\"name\": \"no-resources\"}"); + cfg.put("default.policy.3", "{\"name\": \"valid\", \"description\": \"my desc\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(1, policies.size()); + assertEquals("valid", policies.get(0).getName()); + assertEquals("my desc", policies.get(0).getDescription()); + } + + @Test + public void test18_DefaultPolicyFromJson_NameAndDescriptionFromConfigNameWhenNotGiven() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", "{\"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + cfg.put("default.policy.7", "{\"name\": \" \", \"resources\": {\"database\": {\"values\": [\"db7\"]}}}"); + cfg.put("default.policy.2", "{\"name\": \"named\", \"resources\": {\"database\": {\"values\": [\"db2\"]}}}"); + cfg.put("default.policy.3", "{\"description\": \"my desc\", \"resources\": {\"database\": {\"values\": [\"db3\"]}}}"); + cfg.put("default.policy.audit-db", "{\"resources\": {\"database\": {\"values\": [\"audit\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + Map nameToDescription = svc.getDefaultRangerPolicies().stream().collect(Collectors.toMap(RangerPolicy::getName, RangerPolicy::getDescription)); + Map expected = new HashMap<>(); + expected.put("default.policy.1", "default.policy.1"); + expected.put("default.policy.7", "default.policy.7"); + expected.put("named", "default.policy.2"); + expected.put("default.policy.3", "my desc"); + expected.put("default.policy.audit-db", "default.policy.audit-db"); + assertEquals(expected, nameToDescription); + } + + @Test + public void test13_DefaultPolicyFromJson_IgnoredWhenAdditionalPoliciesSetupDisabled() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("default.policy.1", "{\"name\": \"p-json\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + assertTrue(svc.getDefaultRangerPolicies().isEmpty()); + } + + @Test + public void test14_DefaultPolicyUsersAndGroupsNotTreatedAsPolicies() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.users", "u1,u2"); + cfg.put("default.policy.groups", "g1"); + cfg.put("default.policy.", "{\"name\": \"no-index\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + assertTrue(svc.getDefaultRangerPolicies().isEmpty()); + } + + @Test + public void test15_DefaultPolicyFromJson_WithPerHierarchyPoliciesAndDefaultPolicyUsers() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("default.policy.users", "svcuser1"); + RangerService service = buildService(cfg); + svc.init(def, service); + int hierarchyPolicyCount = svc.getDefaultRangerPolicies().size(); + + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", "{\"name\": \"p1\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}, \"policyItems\": [{\"users\": [\"u1\"], \"accesses\": [{\"type\": \"select\"}]}]}"); + cfg.put("default.policy.2", "{\"name\": \"p2\", \"resources\": {\"database\": {\"values\": [\"db2\"]}}, \"denyPolicyItems\": [{\"users\": [\"u2\"], \"accesses\": [{\"type\": \"update\"}]}]}"); + cfg.put("default.policy.3", "{\"name\": \"p3\", \"isAuditEnabled\": false, \"resources\": {\"database\": {\"values\": [\"db3\"]}}}"); + svc.init(def, buildService(cfg)); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(hierarchyPolicyCount + 3, policies.size()); + Map byName = new HashMap<>(); + policies.forEach(p -> byName.put(p.getName(), p)); + assertTrue(byName.keySet().containsAll(Arrays.asList("p1", "p2", "p3"))); + assertEquals(Collections.singletonList("u1"), byName.get("p1").getPolicyItems().get(0).getUsers()); + assertEquals(Collections.singletonList("u2"), byName.get("p2").getDenyPolicyItems().get(0).getUsers()); + assertFalse(byName.get("p3").getIsAuditEnabled()); + assertTrue(byName.get("all - database, table").getPolicyItems().get(0).getUsers().contains("svcuser1")); + } + + @Test + public void test16_DefaultPolicyFromJson_NonContiguousIndexes() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", "{\"name\": \"p1\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + cfg.put("default.policy.5", "{\"name\": \"p5\", \"resources\": {\"database\": {\"values\": [\"db5\"]}}}"); + cfg.put("default.policy.10", "{\"name\": \"p10\", \"resources\": {\"database\": {\"values\": [\"db10\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(3, policies.size()); + assertTrue(policies.stream().map(RangerPolicy::getName).collect(Collectors.toSet()).containsAll(Arrays.asList("p1", "p5", "p10"))); + } + + @Test + public void test17_DefaultPolicyFromJson_BlankValueIgnoredAndNonNumericKeyAccepted() throws Exception { + DummyService svc = new DummyService(); + RangerServiceDef def = buildServiceDef(); + Map cfg = new LinkedHashMap<>(); + cfg.put("create.default.policy.per.hierarchy", "false"); + cfg.put("setup.additional.default.policies", "true"); + cfg.put("default.policy.1", ""); + cfg.put("default.policy.2", " "); + cfg.put("default.policy.1a", "{\"name\": \"non-numeric\", \"resources\": {\"database\": {\"values\": [\"db1\"]}}}"); + cfg.put("default.policy.3", "{\"name\": \"valid\", \"resources\": {\"database\": {\"values\": [\"db3\"]}}}"); + RangerService service = buildService(cfg); + svc.init(def, service); + + List policies = svc.getDefaultRangerPolicies(); + assertEquals(2, policies.size()); + assertTrue(policies.stream().map(RangerPolicy::getName).collect(Collectors.toSet()).containsAll(Arrays.asList("non-numeric", "valid"))); + } }