diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index dfd295bbce0..bd2d2d05c72 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -28,6 +28,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.model.RangerRole; +import org.apache.ranger.plugin.model.RangerServiceHeaderInfo; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; import org.apache.ranger.plugin.util.JsonUtilsV2; @@ -51,15 +52,18 @@ import java.io.UnsupportedEncodingException; import java.security.PrivilegedExceptionAction; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; public class RangerAdminRESTClient extends AbstractRangerAdminClient { private static final Logger LOG = LoggerFactory.getLogger(RangerAdminRESTClient.class); - private static final TypeReference> TYPE_LIST_STRING = new TypeReference>() {}; + private static final TypeReference> TYPE_LIST_STRING = new TypeReference>() {}; + private static final TypeReference> TYPE_LIST_SERVICE_HEADER = new TypeReference>() {}; private final String pluginCapabilities = Long.toHexString(new RangerPluginCapability().getPluginCapabilities()); private final RangerRESTUtils restUtils = new RangerRESTUtils(); @@ -116,7 +120,7 @@ public void init(String serviceName, String appId, String propertyPrefix, Config url = url.substring(0, url.length() - 1); } - init(url, sslConfigFileName, restClientConnTimeOutMs, restClientReadTimeOutMs, restClientMaxRetryAttempts, restClientRetryIntervalMs, config); + init(url, sslConfigFileName, restClientConnTimeOutMs, restClientReadTimeOutMs, restClientMaxRetryAttempts, restClientRetryIntervalMs, config, propertyPrefix); try { this.serviceNameUrlParam = URLEncoderUtil.encodeURIParam(serviceName); @@ -126,6 +130,59 @@ public void init(String serviceName, String appId, String propertyPrefix, Config } } + public Map> getServiceAccountUsers() throws Exception { + LOG.debug("==> RangerAdminRESTClient.getServiceAccountUsers()"); + + final boolean isSecureMode = isAuthenticationEnabled(); + final Cookie sessionId = this.sessionId; + final Map queryParams = Collections.emptyMap(); + final Response response; + + if (isSecureMode) { + response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction) () -> + restClient.get(RangerRESTUtils.REST_URL_SERVICE_HEADERS, queryParams, sessionId)); + } else { + response = restClient.get(RangerRESTUtils.REST_URL_SERVICE_HEADERS, queryParams, sessionId); + } + + checkAndResetSessionCookie(response); + + Map> ret = Collections.emptyMap(); + + if (response == null) { + throw new Exception("Unknown error while retrieving service-account users"); + } else if (response.getStatus() == HttpStatus.SC_OK) { + List serviceHeaders = JsonUtilsV2.jsonToObj(response.readEntity(String.class), + TYPE_LIST_SERVICE_HEADER); + + if (serviceHeaders != null && !serviceHeaders.isEmpty()) { + Map> serviceAccountUsers = new HashMap<>(); + + for (RangerServiceHeaderInfo serviceHeader : serviceHeaders) { + if (serviceHeader != null && Boolean.TRUE.equals(serviceHeader.getIsEnabled()) && + !StringUtil.isEmpty(serviceHeader.getName()) && serviceHeader.getServiceAccountUsers() != null && + !serviceHeader.getServiceAccountUsers().isEmpty()) { + serviceAccountUsers.put(serviceHeader.getName(), serviceHeader.getServiceAccountUsers()); + } + } + + ret = serviceAccountUsers; + } + } else { + LOG.error("getServiceAccountUsers() failed: HTTP status={}, isSecure={}", response.getStatus(), isSecureMode); + + if (response.getStatus() == HttpStatus.SC_UNAUTHORIZED || response.getStatus() == HttpStatus.SC_FORBIDDEN) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " while retrieving service-account users"); + } + + LOG.debug("<== RangerAdminRESTClient.getServiceAccountUsers(): serviceCount={}", ret.size()); + + return ret; + } + @Override public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated({}, {})", lastKnownVersion, lastActivationTimeInMillis); @@ -1029,10 +1086,10 @@ public void setTokenSupplier(Supplier tokenSupplier) { } } - private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs, int restClientReadTimeOutMs, int restClientMaxRetryAttempts, int restClientRetryIntervalMs, Configuration config) { + private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs, int restClientReadTimeOutMs, int restClientMaxRetryAttempts, int restClientRetryIntervalMs, Configuration config, String propertyPrefix) { LOG.debug("==> RangerAdminRESTClient.init({}, {})", url, sslConfigFileName); - restClient = new RangerRESTClient(url, sslConfigFileName, config); + restClient = new RangerRESTClient(url, sslConfigFileName, config, propertyPrefix); if (tokenSupplier != null) { restClient.setTokenSupplier(tokenSupplier); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java index bcff32a2e6b..7aaff8c6df0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java @@ -43,6 +43,7 @@ public class RangerRESTUtils { public static final String REST_URL_SERVICE_GET_ROLE_INFO = "/service/public/v2/api/roles/name/"; public static final String REST_URL_SERVICE_GRANT_ROLE = "/service/public/v2/api/roles/grant/"; public static final String REST_URL_SERVICE_REVOKE_ROLE = "/service/public/v2/api/roles/revoke/"; + public static final String REST_URL_SERVICE_HEADERS = "/service/public/v2/api/service-headers"; public static final String REST_URL_SERVICE_SERCURE_GET_USER_GROUP_ROLES = "/service/roles/secure/download/"; public static final String REST_URL_SERVICE_GET_USER_GROUP_ROLES = "/service/roles/download/"; public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download/"; diff --git a/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java b/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java index 1ffd1731b95..1f63e562ab4 100644 --- a/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java +++ b/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java @@ -29,6 +29,7 @@ import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.plugin.util.RangerRESTUtils; import org.apache.ranger.plugin.util.RangerRoles; import org.apache.ranger.plugin.util.RangerServiceNotFoundException; import org.apache.ranger.plugin.util.ServiceGdsInfo; @@ -51,8 +52,10 @@ import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; /** * @generated by Cursor @@ -1146,4 +1149,45 @@ public void test42_secureEndpointAccessDisabled_withoutKerberos_usesNonSecureUrl misc.verify(() -> MiscUtil.executePrivilegedAction(Mockito.>any()), Mockito.never()); } } + + @Test + public void test43_getServiceAccountUsers() throws Exception { + RangerAdminRESTClient client = new RangerAdminRESTClient(); + Configuration cfg = new Configuration(false); + cfg.set("p.policy.rest.url", "http://localhost:6080"); + cfg.setBoolean("p.forceSecureEndpointAccess", false); + client.init("svc", "app", "p", cfg); + RangerRESTClient rest = Mockito.mock(RangerRESTClient.class); + setPrivateField(client, "restClient", rest); + + Response response = mockResponse(200, + "[{\"name\":\"svc1\",\"isEnabled\":true,\"serviceAccountUsers\":[\"alice\"]}," + + "{\"name\":\"svc2\",\"isEnabled\":false,\"serviceAccountUsers\":[\"bob\"]}," + + "{\"name\":\"svc3\",\"isEnabled\":true}]", + Collections.emptyList()); + Mockito.when(rest.get(Mockito.anyString(), Mockito.anyMap(), Mockito.any())).thenReturn(response); + + Map> users = client.getServiceAccountUsers(); + + Assertions.assertEquals(Collections.singleton("alice"), users.get("svc1")); + Assertions.assertEquals(1, users.size()); + Mockito.verify(rest).get(Mockito.eq(RangerRESTUtils.REST_URL_SERVICE_HEADERS), Mockito.anyMap(), Mockito.any()); + } + + @Test + public void test44_getServiceAccountUsers_forbidden() throws Exception { + RangerAdminRESTClient client = new RangerAdminRESTClient(); + Configuration cfg = new Configuration(false); + cfg.set("p.policy.rest.url", "http://localhost:6080"); + cfg.setBoolean("p.forceSecureEndpointAccess", false); + client.init("svc", "app", "p", cfg); + RangerRESTClient rest = Mockito.mock(RangerRESTClient.class); + setPrivateField(client, "restClient", rest); + Response response = Mockito.mock(Response.class); + Mockito.when(response.getStatus()).thenReturn(403); + Mockito.when(rest.get(Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenReturn(response); + + Assertions.assertThrows(AccessControlException.class, client::getServiceAccountUsers); + } } diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index bcd3c955f57..fc9b290031a 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -27,6 +27,7 @@ private AuditServerConstants() {} public static final String PROP_PREFIX_AUDIT_SERVER_SERVICE = PROP_PREFIX_AUDIT_SERVER + "service."; public static final String PROP_AUTH_TO_LOCAL = PROP_PREFIX_AUDIT_SERVER + "auth.to.local"; public static final String PROP_SUFFIX_ALLOWED_USERS = ".allowed.users"; + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_PREFIX_AUDIT_SERVER + "serviceaccount.refresher.enabled"; public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "service.kerberos.principal"; public static final String PROP_AUDIT_SERVICE_KEYTAB = "service.kerberos.keytab"; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 37c2fce39af..d4fd97d98f5 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -20,7 +20,9 @@ package org.apache.ranger.audit.rest; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.producer.AuditDestinationMgr; import org.apache.ranger.audit.provider.MiscUtil; @@ -43,10 +45,14 @@ import javax.ws.rs.core.Response; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.stream.Collectors; import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE; @@ -58,13 +64,109 @@ public class AuditREST { private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); - private static final Map> allowedServiceUsers; + private static final String RANGER_ADMIN_CONFIG_PREFIX = "ranger.audit.ingestor"; + private static final String PROP_POLICY_REST_URL = RANGER_ADMIN_CONFIG_PREFIX + ".policy.rest.url"; + private static final String PROP_POLICY_POLL_INTERVAL_MS = RANGER_ADMIN_CONFIG_PREFIX + ".policy.pollIntervalMs"; + private static final long DEFAULT_POLL_INTERVAL_MS = 30 * 1000L; + + private static final Map> configuredAllowedUsers; + + private static volatile Map> allowedServiceUsers; + private static RangerAdminRESTClient rangerAdminClient; + private static Timer serviceAccountUsersRefreshTimer; static { - allowedServiceUsers = initializeAllowedUsers(); + configuredAllowedUsers = initializeAllowedUsers(); + allowedServiceUsers = mergeAllowedUsers(configuredAllowedUsers, Collections.emptyMap()); initializeAuthToLocal(); } + public static synchronized void startServiceAccountUsersRefresh(Configuration configuration) { + if (!configuration.getBoolean(AuditServerConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, false)) { + return; + } + + String rangerAdminUrl = configuration.get(PROP_POLICY_REST_URL); + + if (serviceAccountUsersRefreshTimer == null && StringUtils.isNotBlank(rangerAdminUrl)) { + try { + RangerAdminRESTClient client = new RangerAdminRESTClient(); + + client.init("audit-ingestor", "ranger-audit", RANGER_ADMIN_CONFIG_PREFIX, configuration); + + rangerAdminClient = client; + + long pollIntervalMs = configuration.getLong(PROP_POLICY_POLL_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS); + Timer timer = new Timer("auditServiceAccountUsersRefresh", true); + + timer.schedule(new TimerTask() { + @Override + public void run() { + refreshAllowedUsers(rangerAdminClient); + } + }, pollIntervalMs, pollIntervalMs); + + serviceAccountUsersRefreshTimer = timer; + + LOG.info("Scheduled service account user refresh every {} ms from {}", pollIntervalMs, rangerAdminUrl); + refreshAllowedUsers(client); + } catch (Exception e) { + LOG.warn("Failed to start service account user refresh", e); + } + } + } + + public static synchronized void stopServiceAccountUsersRefresh() { + Timer timer = serviceAccountUsersRefreshTimer; + + serviceAccountUsersRefreshTimer = null; + + if (timer != null) { + timer.cancel(); + } + } + + static void refreshAllowedUsers(RangerAdminRESTClient client) { + if (client != null) { + try { + Map> serviceAccountUsers = client.getServiceAccountUsers(); + allowedServiceUsers = mergeAllowedUsers(configuredAllowedUsers, serviceAccountUsers); + if (!serviceAccountUsers.isEmpty()) { + LOG.info("Merged service.account.users for {} service(s) from Ranger Admin", serviceAccountUsers.size()); + } + } catch (Exception e) { + LOG.warn("Failed to refresh service account users from Ranger Admin", e); + } + } + } + + static Map> mergeAllowedUsers(Map> configuredAllowedUsers, Map> serviceAccountUsersByServiceName) { + Map> ret = new HashMap<>(); + + for (Map.Entry> entry : configuredAllowedUsers.entrySet()) { + ret.put(entry.getKey(), new HashSet<>(entry.getValue())); + } + + for (Map.Entry> entry : serviceAccountUsersByServiceName.entrySet()) { + String serviceName = entry.getKey(); + Set serviceAccountUsers = entry.getValue(); + + if (StringUtils.isNotBlank(serviceName) && serviceAccountUsers != null && !serviceAccountUsers.isEmpty()) { + Set allowedUsers = ret.get(serviceName); + + if (allowedUsers == null) { + allowedUsers = new HashSet<>(); + + ret.put(serviceName, allowedUsers); + } + + allowedUsers.addAll(serviceAccountUsers); + } + } + + return ret; + } + @Autowired AuditDestinationMgr auditDestinationMgr; @@ -285,6 +387,10 @@ private String applyAuthToLocal(String principal) { return principal; } + if (principal.startsWith("spiffe://")) { + return principal; + } + // Check if this looks like a Kerberos principal (has @ or /) if (!principal.contains("@") && !principal.contains("/")) { LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); @@ -342,10 +448,6 @@ private boolean isAllowedServiceUser(String serviceName, String userName) { return ret; } - /** - * Initialize the set of allowed service users from configuration - * @return Map of allowed usernames per service - */ private static Map> initializeAllowedUsers() { Map> ret = new HashMap<>(); AuditServerConfig config = AuditServerConfig.getInstance(); diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java new file mode 100644 index 00000000000..2c36638f0be --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.rest; + +import org.apache.ranger.audit.server.AuditServerConfig; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +/** + * Starts service-account refresh from the Spring webapp class loader so {@link AuditREST} + * static allow maps stay in sync with the JAX-RS beans that authorize ingest traffic. + */ +@Component +public class AuditServiceAccountRefreshBootstrap { + @PostConstruct + public void start() { + AuditREST.startServiceAccountUsersRefresh(AuditServerConfig.getInstance()); + } + + @PreDestroy + public void stop() { + AuditREST.stopServiceAccountUsersRefresh(); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java index cde2b8267aa..c3018e53ac6 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider; import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.rest.AuditREST; import org.apache.ranger.audit.utils.AuditServerLogFormatter; import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider; import org.apache.tomcat.util.net.SSLHostConfig; @@ -473,6 +474,8 @@ private void addShutdownHook() { private void gracefulShutdown() { LOG.info("==> EmbeddedServer.gracefulShutdown()"); + AuditREST.stopServiceAccountUsersRefresh(); + // Instead of trying to manually shutdown Spring context, let's rely on Tomcat's // normal webapp lifecycle which will trigger ContextLoaderListener.contextDestroyed() // and call @PreDestroy methods automatically diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index d3bf921cff0..b72aa8a0dd1 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -205,6 +205,42 @@ Comma-separated list of HTTP header name(s) carrying a SPIFFE ID (e.g. spiffe://cluster/ns/<ns>/sa/<service-account>). A single header name is also valid. The full SPIFFE ID is used as the authenticated principal. When multiple headers are configured, the first one carrying a well-formed SPIFFE ID is used. Consulted only when the username header is absent. + + ranger.audit.ingestor.serviceaccount.refresher.enabled + false + When true and ranger.audit.ingestor.policy.rest.url is set, periodically merge service.account.users from Ranger Admin into allowed audit callers. Override via Cloudera Manager safety valve. + + + + ranger.audit.ingestor.policy.rest.url + + Ranger Admin base URL for service-account refresh. Uses ranger.audit.ingestor.policy.rest.ssl.config.file and .client.username/.password for authentication (safety-valve overrides). Leave empty to use only ranger.audit.ingestor.service.<serviceName>.allowed.users. Poll identity must be in ranger.admin.service.account.info.viewers. + + + + ranger.audit.ingestor.policy.pollIntervalMs + 30000 + How often (ms) to refresh service.account.users when ranger.audit.ingestor.serviceaccount.refresher.enabled is true. + + + + ranger.audit.ingestor.policy.rest.ssl.config.file + + SSL config file for the Ranger Admin connection. Required when ranger.audit.ingestor.policy.rest.url uses https. + + + + ranger.audit.ingestor.policy.rest.client.username + + Username for the Ranger Admin connection (basic auth). + + + + ranger.audit.ingestor.policy.rest.client.password + + Password for the Ranger Admin connection (basic auth). + + ranger.audit.ingestor.service.dev_hdfs.allowed.users hdfs diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java new file mode 100644 index 00000000000..58ea6eb6552 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.rest; + +import org.apache.ranger.admin.client.RangerAdminRESTClient; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class AuditRESTTest { + @Test + void testMergeAllowedUsersUnionsServiceAccountUsers() { + Map> configuredAllowedUsers = new HashMap<>(); + + configuredAllowedUsers.put("dev_hive", new HashSet<>(Collections.singleton("hive"))); + + Map> serviceAccountUsersByServiceName = new HashMap<>(); + + serviceAccountUsersByServiceName.put("dev_hive", Collections.singleton("hive2")); + serviceAccountUsersByServiceName.put("dev_kafka", Collections.singleton("kafka")); + + Map> mergedAllowedUsers = AuditREST.mergeAllowedUsers(configuredAllowedUsers, serviceAccountUsersByServiceName); + + assertEquals(new HashSet<>(Arrays.asList("hive", "hive2")), mergedAllowedUsers.get("dev_hive")); + assertEquals(Collections.singleton("kafka"), mergedAllowedUsers.get("dev_kafka")); + } + + @Test + void testRefreshKeepsAllowedUsersWhenAdminCallFails() throws Exception { + Map> before = allowedUsers(); + + AuditREST.refreshAllowedUsers(new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() throws Exception { + throw new Exception("admin down"); + } + }); + + assertSame(before, allowedUsers()); + } + + @Test + void testRefreshAddsServiceAccountUsers() throws Exception { + Map> before = allowedUsers(); + + try { + AuditREST.refreshAllowedUsers(new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.singletonMap("dev_new", Collections.singleton("new-user")); + } + }); + + assertTrue(allowedUsers().get("dev_new").contains("new-user")); + } finally { + AuditREST.refreshAllowedUsers(new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.emptyMap(); + } + }); + + assertEquals(before.keySet(), allowedUsers().keySet()); + } + } + + @SuppressWarnings("unchecked") + private static Map> allowedUsers() throws Exception { + Field field = AuditREST.class.getDeclaredField("allowedServiceUsers"); + + field.setAccessible(true); + + return (Map>) field.get(null); + } +} diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java new file mode 100644 index 00000000000..b92f348e1ff --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.rest; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class AuditServiceAccountRefreshBootstrapTest { + @Test + void lifecycleWithRefresherDisabledDoesNotThrow() { + AuditServiceAccountRefreshBootstrap bootstrap = new AuditServiceAccountRefreshBootstrap(); + + assertDoesNotThrow(() -> { + bootstrap.start(); + bootstrap.stop(); + }); + } +} diff --git a/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml b/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml index 4a5c3d47801..cef28367bd8 100644 --- a/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml +++ b/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml @@ -29,6 +29,7 @@ ranger.audit.ingestor.contextName: / ranger.audit.ingestor.host: ranger-audit-ingestor.rangernw ranger.audit.ingestor.http.port: '7081' ranger.audit.ingestor.https.port: '7182' +ranger.serviceaccount.refresher.policy.rest.url: 'http://ranger:6080' ranger.audit.ingestor.https.attrib.ssl.enabled: 'false' # HTTPS/SSL Configuration diff --git a/functional-tests/audit_ingestor/audit_ingestor_utils.py b/functional-tests/audit_ingestor/audit_ingestor_utils.py new file mode 100644 index 00000000000..74ae2ac9ac9 --- /dev/null +++ b/functional-tests/audit_ingestor/audit_ingestor_utils.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest +import requests + +from ranger_test_utils.utils import ( + DEFAULT_JSON_HEADERS, + SERVICE_URL, + assert_http_ok, + create_ranger_admin_session, + unique_suffix, +) + +AUDIT_INGESTOR_BASE_URL = os.environ.get( + "RANGER_AUDIT_INGESTOR_BASE_URL", "http://localhost:7081/api/audit" +).rstrip("/") +SERVICE_NAME = "dev_hive" +SERVICE_ACCOUNT_USERS = "service.account.users" +REFRESH_TIMEOUT_SEC = int( + os.environ.get("AUDIT_SERVICE_ACCOUNT_REFRESH_TIMEOUT_SEC", "90") +) + + +class AuditIngestorServiceAccountTestContext: + def __init__(self): + self.admin = create_ranger_admin_session() + self.audit_ingestor = requests.Session() + self.audit_ingestor.headers.update(DEFAULT_JSON_HEADERS) + self.service = None + self.original_value = None + self.user = "audit-functional-" + unique_suffix(8) + self.unauthorized_user = "audit-unauthorized-" + unique_suffix(8) + + def setup(self): + service_url = SERVICE_URL + "name/" + SERVICE_NAME + response = self.admin.get(service_url) + assert_http_ok(response, 200, "Get service " + SERVICE_NAME, service_url) + + self.service = response.json() + configs = self.service.setdefault("configs", {}) + self.original_value = configs.get(SERVICE_ACCOUNT_USERS) + original_users = [ + user.strip() + for user in (self.original_value or "").split(",") + if user.strip() + ] + configs[SERVICE_ACCOUNT_USERS] = ",".join(original_users + [self.user]) + + update_url = SERVICE_URL + str(self.service["id"]) + updated = self.admin.put(update_url, json=self.service) + assert_http_ok( + updated, 200, "Add dynamic service-account user", update_url + ) + self._wait_for_status(200) + + return self + + def cleanup(self): + if self.service is not None: + configs = self.service["configs"] + if self.original_value is None: + configs.pop(SERVICE_ACCOUNT_USERS, None) + else: + configs[SERVICE_ACCOUNT_USERS] = self.original_value + + update_url = SERVICE_URL + str(self.service["id"]) + restored = self.admin.put(update_url, json=self.service) + assert_http_ok( + restored, 200, "Restore service-account users", update_url + ) + self._wait_for_status(403) + + self.audit_ingestor.close() + self.admin.close() + + def post_access( + self, user=None, service_name=SERVICE_NAME, service_type="hive" + ): + self.audit_ingestor.cookies.clear() + return self.audit_ingestor.post( + AUDIT_INGESTOR_BASE_URL + "/access", + params={ + "serviceName": service_name, + "appId": "service-account-functional-test", + }, + headers={"X-Forwarded-User": user or self.user}, + json=[ + { + "repoName": service_name, + "serviceType": service_type, + "user": "audit-functional-target", + "accessType": "select", + "resourcePath": "default/sales", + "accessResult": 1, + "action": "select", + } + ], + timeout=30, + ) + + def _wait_for_status(self, expected_status): + deadline = time.monotonic() + REFRESH_TIMEOUT_SEC + response = None + + while time.monotonic() < deadline: + response = self.post_access() + if response.status_code == expected_status: + return + time.sleep(2) + + pytest.fail( + "Audit Ingestor did not return HTTP " + + str(expected_status) + + " for the dynamic service-account user; last response was " + + ( + str(response.status_code) + ": " + response.text + if response is not None + else "none" + ) + ) diff --git a/functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py b/functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py new file mode 100644 index 00000000000..f5f7414981c --- /dev/null +++ b/functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from audit_ingestor.audit_ingestor_utils import ( + AuditIngestorServiceAccountTestContext, +) +from ranger_test_utils.utils import assert_http_ok + +pytestmark = pytest.mark.audit_ingestor + + +@pytest.fixture(scope="module") +def audit_ingestor_context(): + context = AuditIngestorServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def test_unauthorized_user_cannot_post_audits(audit_ingestor_context): + response = audit_ingestor_context.post_access( + user=audit_ingestor_context.unauthorized_user + ) + print("\n[Audit Ingestor] Unauthorized /access returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Unauthorized POST /access") + + +def test_service_account_user_cannot_post_for_another_service( + audit_ingestor_context, +): + response = audit_ingestor_context.post_access( + service_name="dev_hdfs", service_type="hdfs" + ) + print("\n[Audit Ingestor] Cross-service /access returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Cross-service POST /access") + + +def test_configured_user_can_post_audits(audit_ingestor_context): + response = audit_ingestor_context.post_access(user="hive") + print("\n[Audit Ingestor] Configured-user /access returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 200, "Configured-user POST /access") diff --git a/functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py b/functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py new file mode 100644 index 00000000000..f1634b0c63c --- /dev/null +++ b/functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from audit_ingestor.audit_ingestor_utils import ( + AuditIngestorServiceAccountTestContext, +) +from ranger_test_utils.utils import ( + assert_http_ok, + get_test_logger, + log_http_response, +) + +pytestmark = pytest.mark.audit_ingestor +logger = get_test_logger(__name__) + + +@pytest.fixture(scope="module") +def audit_ingestor_context(): + context = AuditIngestorServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def test_dynamic_service_account_user_can_post_audits(audit_ingestor_context): + print( + "\n[Audit Ingestor] Testing POST /access " + "with a dynamic service-account user" + ) + response = audit_ingestor_context.post_access() + print( + "[Audit Ingestor] POST /access returned HTTP " + + str(response.status_code) + ) + log_http_response(logger, "POST /access", response) + assert_http_ok(response, 200, "POST /access") + + result = response.json() + assert result["serviceName"] == "dev_hive" + assert result["total"] == 1 diff --git a/functional-tests/pdp/pdp_utils.py b/functional-tests/pdp/pdp_utils.py new file mode 100644 index 00000000000..1ccd6083fac --- /dev/null +++ b/functional-tests/pdp/pdp_utils.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest +import requests + +from ranger_test_utils.utils import ( + DEFAULT_JSON_HEADERS, + SERVICE_URL, + assert_http_ok, + create_ranger_admin_session, + unique_suffix, +) + +PDP_BASE_URL = os.environ.get( + "RANGER_PDP_BASE_URL", "http://localhost:6500/authz/v1" +).rstrip("/") +SERVICE_NAME = "dev_hive" +SERVICE_ACCOUNT_USERS = "service.account.users" +REFRESH_TIMEOUT_SEC = int( + os.environ.get("PDP_SERVICE_ACCOUNT_REFRESH_TIMEOUT_SEC", "90") +) + + +class PdpServiceAccountTestContext: + def __init__(self): + self.admin = create_ranger_admin_session() + self.pdp = requests.Session() + self.pdp.headers.update(DEFAULT_JSON_HEADERS) + self.service = None + self.original_value = None + self.user = "pdp-functional-" + unique_suffix(8) + self.unauthorized_user = "pdp-unauthorized-" + unique_suffix(8) + + def setup(self): + service_url = SERVICE_URL + "name/" + SERVICE_NAME + response = self.admin.get(service_url) + assert_http_ok(response, 200, "Get service " + SERVICE_NAME, service_url) + + self.service = response.json() + configs = self.service.setdefault("configs", {}) + self.original_value = configs.get(SERVICE_ACCOUNT_USERS) + original_users = [ + user.strip() + for user in (self.original_value or "").split(",") + if user.strip() + ] + configs[SERVICE_ACCOUNT_USERS] = ",".join(original_users + [self.user]) + + update_url = SERVICE_URL + str(self.service["id"]) + updated = self.admin.put(update_url, json=self.service) + assert_http_ok( + updated, 200, "Add dynamic service-account user", update_url + ) + self._wait_for_status(200) + + return self + + def cleanup(self): + if self.service is not None: + configs = self.service["configs"] + if self.original_value is None: + configs.pop(SERVICE_ACCOUNT_USERS, None) + else: + configs[SERVICE_ACCOUNT_USERS] = self.original_value + + update_url = SERVICE_URL + str(self.service["id"]) + restored = self.admin.put(update_url, json=self.service) + assert_http_ok( + restored, 200, "Restore service-account users", update_url + ) + self._wait_for_status(403) + + self.pdp.close() + self.admin.close() + + def post(self, endpoint, payload, user=None): + return self.pdp.post( + PDP_BASE_URL + endpoint, + headers={"X-Forwarded-User": user or self.user}, + json=payload, + timeout=30, + ) + + @staticmethod + def context(service_name=SERVICE_NAME, service_type="hive"): + return {"serviceType": service_type, "serviceName": service_name} + + @staticmethod + def access(resource): + return { + "resource": {"name": resource}, + "action": "QUERY", + "permissions": ["select"], + } + + def _wait_for_status(self, expected_status): + deadline = time.monotonic() + REFRESH_TIMEOUT_SEC + response = None + + while time.monotonic() < deadline: + response = self.post( + "/authorize", + { + "requestId": "service-account-refresh-check", + "user": {"name": "hive"}, + "access": self.access("table:default/sales"), + "context": self.context(), + }, + ) + if response.status_code == expected_status: + return + time.sleep(2) + + pytest.fail( + "PDP did not return HTTP " + + str(expected_status) + + " for the dynamic service-account user; last response was " + + ( + str(response.status_code) + ": " + response.text + if response is not None + else "none" + ) + ) diff --git a/functional-tests/pdp/test_pdp_rest_authorization_scenarios.py b/functional-tests/pdp/test_pdp_rest_authorization_scenarios.py new file mode 100644 index 00000000000..8d8af6973c4 --- /dev/null +++ b/functional-tests/pdp/test_pdp_rest_authorization_scenarios.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pdp.pdp_utils import PdpServiceAccountTestContext +from ranger_test_utils.utils import assert_http_ok + +pytestmark = pytest.mark.pdp + + +@pytest.fixture(scope="module") +def pdp_context(): + context = PdpServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def test_unauthorized_user_cannot_authorize(pdp_context): + response = pdp_context.post( + "/authorize", + { + "requestId": "pdp-functional-unauthorized", + "user": {"name": "hive"}, + "access": pdp_context.access("table:default/sales"), + "context": pdp_context.context(), + }, + pdp_context.unauthorized_user, + ) + print("\n[PDP] Unauthorized /authorize returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Unauthorized POST /authorize") + + +def test_service_account_user_cannot_authorize_another_service(pdp_context): + response = pdp_context.post( + "/authorizeMulti", + { + "requestId": "pdp-functional-wrong-service", + "user": {"name": "hdfs"}, + "accesses": [pdp_context.access("/tmp/test")], + "context": pdp_context.context("dev_hdfs", "hdfs"), + }, + ) + print("\n[PDP] Cross-service /authorizeMulti returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Cross-service POST /authorizeMulti") + + +def test_configured_user_can_get_permissions(pdp_context): + response = pdp_context.post( + "/permissions", + { + "requestId": "pdp-functional-configured-user", + "resource": {"name": "table:default/sales"}, + "context": pdp_context.context(), + }, + "ranger", + ) + print("\n[PDP] Configured-user /permissions returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 200, "Configured-user POST /permissions") diff --git a/functional-tests/pdp/test_pdp_rest_service_account_users.py b/functional-tests/pdp/test_pdp_rest_service_account_users.py new file mode 100644 index 00000000000..3f029e7d7e2 --- /dev/null +++ b/functional-tests/pdp/test_pdp_rest_service_account_users.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pdp.pdp_utils import PdpServiceAccountTestContext +from ranger_test_utils.utils import ( + assert_http_ok, + get_test_logger, + log_http_response, +) + +pytestmark = pytest.mark.pdp +logger = get_test_logger(__name__) + + +@pytest.fixture(scope="module") +def pdp_context(): + context = PdpServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def post(pdp_context, endpoint, payload): + print("\n[PDP] Testing POST " + endpoint + " with a dynamic service-account user") + response = pdp_context.post(endpoint, payload) + print("[PDP] POST " + endpoint + " returned HTTP " + str(response.status_code)) + log_http_response(logger, "POST " + endpoint, response) + assert_http_ok(response, 200, "POST " + endpoint) + + return response.json() + + +def test_dynamic_service_account_user_can_authorize(pdp_context): + result = post( + pdp_context, + "/authorize", + { + "requestId": "pdp-functional-authorize", + "user": {"name": "hive"}, + "access": pdp_context.access("table:default/sales"), + "context": pdp_context.context(), + }, + ) + + assert result["requestId"] == "pdp-functional-authorize" + assert result["decision"] in ("ALLOW", "DENY") + + +def test_dynamic_service_account_user_can_authorize_multi(pdp_context): + result = post( + pdp_context, + "/authorizeMulti", + { + "requestId": "pdp-functional-authorize-multi", + "user": {"name": "hive"}, + "accesses": [ + pdp_context.access("table:default/sales"), + pdp_context.access("table:default/customers"), + ], + "context": pdp_context.context(), + }, + ) + + assert result["requestId"] == "pdp-functional-authorize-multi" + assert len(result["accesses"]) == 2 + + +def test_dynamic_service_account_user_can_get_permissions(pdp_context): + result = post( + pdp_context, + "/permissions", + { + "requestId": "pdp-functional-permissions", + "resource": {"name": "table:default/sales"}, + "context": pdp_context.context(), + }, + ) + + assert result["resource"]["name"] == "table:default/sales" + + +def test_dynamic_service_account_user_can_filter_resources(pdp_context): + result = post( + pdp_context, + "/filterResources", + { + "requestId": "pdp-functional-filter-resources", + "user": {"name": "hive"}, + "resources": [ + {"name": "table:default/sales"}, + {"name": "table:default/customers"}, + ], + "action": "select", + "permissions": ["select"], + "context": pdp_context.context(), + }, + ) + + assert result["requestId"] == "pdp-functional-filter-resources" + assert "resources" in result diff --git a/functional-tests/pytest.ini b/functional-tests/pytest.ini index d17dbeb90db..feb8b3dd780 100644 --- a/functional-tests/pytest.ini +++ b/functional-tests/pytest.ini @@ -32,6 +32,8 @@ markers = hbase: HBase plugin enforcement (Docker HBase shell) hdfs_policy: HDFS Ranger plugin path policy enforcement (Docker dfs) kafka: Kafka plugin enforcement (Docker Kafka console producer) + pdp: PDP live REST API tests + audit_ingestor: Audit Ingestor live REST API tests pythonpath = . # No console logging during pytest; logger.info -> logs/.log (see conftest.py). log_cli = false diff --git a/functional-tests/readme.md b/functional-tests/readme.md index b11915cce4b..b50a0e7f59e 100644 --- a/functional-tests/readme.md +++ b/functional-tests/readme.md @@ -36,6 +36,8 @@ A Docker-based test execution framework that validates Apache Ranger service int | **hbase** | HBase Ranger plugin enforcement via HBase shell (requires `ranger-hadoop` + `ranger-hbase`) | | **jvm_locale_test** | Ranger Admin JVM locale tests. Run with `run-jvm-locale-tests.sh`. | | **kafka** | Kafka Ranger plugin enforcement via console producer (requires `ranger-kafka`) | +| **pdp** | Live PDP REST authorization using dynamically refreshed service-account users | +| **audit_ingestor** | Live Audit Ingestor REST authorization using dynamically refreshed service-account users | ### Directory Structure diff --git a/pdp/conf.dist/ranger-pdp-site.xml b/pdp/conf.dist/ranger-pdp-site.xml index 6f3152f4157..88ac3a6ad68 100644 --- a/pdp/conf.dist/ranger-pdp-site.xml +++ b/pdp/conf.dist/ranger-pdp-site.xml @@ -230,12 +230,20 @@ Policy source implementation. Use RangerAdminRESTClient to download policies live from Ranger Admin. + + ranger.pdp.serviceaccount.refresher.enabled + false + When true and ranger.authz.default.policy.rest.url is set, periodically merge service.account.users from Ranger Admin into PDP delegation users. Override via Cloudera Manager safety valve. Poll credentials use ranger.authz.default.policy.rest.* (same as policy download). + + ranger.authz.default.policy.rest.url http://localhost:6080 URL of the Ranger Admin server. Comma-separated for HA. Example: http://ranger-admin-1:6080,http://ranger-admin-2:6080 + Also used for service-account user refresh when ranger.pdp.serviceaccount.refresher.enabled is true (rest client auth via ranger.authz.default.policy.rest.ssl.config.file and .client.username/.password). + The REST client identity must be listed in ranger.admin.service.account.info.viewers on Admin. @@ -268,7 +276,7 @@ ranger.authz.default.policy.pollIntervalMs 30000 - How often (ms) to poll Ranger Admin for updated policies and roles. + How often (ms) to poll Ranger Admin for updated policies and roles and to refresh service-account users when ranger.pdp.serviceaccount.refresher.enabled is true. diff --git a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java index d541d426888..13f3497709b 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java @@ -52,7 +52,6 @@ public class RangerPdpConfig { private static final String DEFAULT_CONFIG_FILE = "ranger-pdp-default.xml"; private static final String SITE_CONFIG_FILE = "ranger-pdp-site.xml"; - private final Properties props = new Properties(); public RangerPdpConfig() { @@ -273,7 +272,8 @@ private void parseHadoopXml(InputStream in, String source) { */ private void applySystemPropertyOverrides() { for (String key : System.getProperties().stringPropertyNames()) { - if (key.startsWith(RangerPdpConstants.PROP_PDP_PREFIX) || key.startsWith(RangerPdpConstants.PROP_AUTHZ_PREFIX)) { + if (key.startsWith(RangerPdpConstants.PROP_PDP_PREFIX) || + key.startsWith(RangerPdpConstants.PROP_AUTHZ_PREFIX)) { props.setProperty(key, System.getProperty(key)); } } diff --git a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java index a0c40f29020..d695aa7de1d 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java @@ -87,7 +87,7 @@ private RangerPdpConstants() { public static final String PROP_AUTHZ_PREFIX = "ranger.authz."; public static final String PROP_PDP_PREFIX = "ranger.pdp."; public static final String PROP_PDP_SERVICE_PREFIX = PROP_PDP_PREFIX + "service."; - + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_PDP_PREFIX + "serviceaccount.refresher.enabled"; // delegation users public static final String PROP_SUFFIX_DELEGATION_USERS = ".delegation.users"; public static final String WILDCARD_SERVICE_NAME = "*"; diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java new file mode 100644 index 00000000000..fe1948c5048 --- /dev/null +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.pdp.rest; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Starts {@link RangerPdpREST} service-account refresh from the JAX-RS / HK2 runtime + * (same lifecycle as delegated authorization checks), mirroring audit ingestor's bootstrap. + */ +@Singleton +public class PdpServiceAccountRefreshBootstrap { + private final RangerPdpREST rangerPdpREST; + + @Inject + public PdpServiceAccountRefreshBootstrap(RangerPdpREST rangerPdpREST) { + this.rangerPdpREST = rangerPdpREST; + } + + @PostConstruct + public void start() { + rangerPdpREST.startServiceAccountUsersRefresh(); + } + + @PreDestroy + public void stop() { + rangerPdpREST.stopServiceAccountUsersRefresh(); + } +} diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java index 42371b452d0..ecaaee48fb4 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java @@ -35,6 +35,7 @@ *
  • {@link AuthorizerBinder} – HK2 binder that makes the {@link RangerAuthorizer} * instance injectable into {@link RangerPdpREST} via {@code @Inject} *
  • {@link RangerPdpREST} – the JAX-RS resource class + *
  • {@link PdpServiceAccountRefreshBootstrap} – service-account refresh lifecycle * */ public class RangerPdpApplication extends ResourceConfig { @@ -42,6 +43,7 @@ public RangerPdpApplication(RangerAuthorizer authorizer, RangerPdpConfig config) register(JacksonFeature.class); register(new AuthorizerBinder(authorizer, config)); register(RangerPdpREST.class); + register(PdpServiceAccountRefreshBootstrap.class); } private static class AuthorizerBinder extends AbstractBinder { diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java index 99b23e2cad4..ae509627b04 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java @@ -22,8 +22,11 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authz.api.RangerAuthorizer; import org.apache.ranger.authz.api.RangerAuthzException; +import org.apache.ranger.authz.embedded.RangerAuthzConfig; import org.apache.ranger.authz.model.RangerAccessInfo; import org.apache.ranger.authz.model.RangerAuthzRequest; import org.apache.ranger.authz.model.RangerAuthzResult; @@ -56,11 +59,15 @@ import javax.ws.rs.core.Response; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.stream.Collectors; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; @@ -70,6 +77,7 @@ import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static org.apache.ranger.authz.api.RangerAuthzApiErrorCode.INVALID_REQUEST_USER_INFO_MISSING; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_PDP_SERVICE_PREFIX; +import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SUFFIX_DELEGATION_USERS; import static org.apache.ranger.pdp.config.RangerPdpConstants.WILDCARD_SERVICE_NAME; @@ -101,7 +109,15 @@ public class RangerPdpREST { private static final Response RESPONSE_OK = Response.ok().build(); - private final Map> delegationUsersByService = new HashMap<>(); + private static final String PROP_POLICY_REST_URL = RangerAuthzConfig.PROP_PREFIX_DEFAULT + "policy.rest.url"; + private static final String PROP_POLICY_POLL_INTERVAL_MS = RangerAuthzConfig.PROP_PREFIX_DEFAULT + "policy.pollIntervalMs"; + private static final long DEFAULT_POLL_INTERVAL_MS = 30 * 1000L; + + private final Map> configuredDelegationUsers = new HashMap<>(); + + private volatile Map> delegationUsersByService = Collections.emptyMap(); + private RangerAdminRESTClient rangerAdminClient; + private Timer serviceAccountUsersRefreshTimer; @Inject private RangerAuthorizer authorizer; @@ -114,7 +130,24 @@ public class RangerPdpREST { @PostConstruct public void initialize() { - initializeDelegationUsers(); + initializeDelegationUsersOnly(); + } + + /** + * Starts Admin {@code service.account.users} polling. Invoked from {@link PdpServiceAccountRefreshBootstrap} + * after {@link #initialize()} has loaded static delegation config. + */ + public synchronized void startServiceAccountUsersRefresh() { + Properties properties = config != null ? config.getAuthzProperties() : new Properties(); + startServiceAccountUsersRefresh(properties); + } + + public synchronized void stopServiceAccountUsersRefresh() { + Timer timer = serviceAccountUsersRefreshTimer; + serviceAccountUsersRefreshTimer = null; + if (timer != null) { + timer.cancel(); + } } /** @@ -466,17 +499,17 @@ private void recordRequestMetrics(Response ret, long startNanos, HttpServletRequ private boolean isDelegationUserForService(String serviceName, String userName) { final boolean ret; - Map> delegationUsersByService = this.delegationUsersByService; + Map> currentDelegationUsersByService = delegationUsersByService; - if (delegationUsersByService.isEmpty()) { + if (currentDelegationUsersByService.isEmpty()) { ret = false; } else if (StringUtils.isBlank(serviceName) || StringUtils.isBlank(userName)) { ret = false; } else { - Set delegationUsers = delegationUsersByService.get(serviceName); + Set delegationUsers = currentDelegationUsersByService.get(serviceName); if (delegationUsers == null) { - delegationUsers = delegationUsersByService.get(RangerPdpConstants.WILDCARD_SERVICE_NAME); + delegationUsers = currentDelegationUsersByService.get(RangerPdpConstants.WILDCARD_SERVICE_NAME); } ret = delegationUsers != null && delegationUsers.contains(userName); @@ -487,7 +520,7 @@ private boolean isDelegationUserForService(String serviceName, String userName) return ret; } - private void initializeDelegationUsers() { + private void initializeDelegationUsersOnly() { Properties properties = config != null ? config.getAuthzProperties() : new Properties(); for (String key : properties.stringPropertyNames()) { @@ -504,21 +537,118 @@ private void initializeDelegationUsers() { .collect(Collectors.toSet()); if (!delegationUsers.isEmpty()) { - delegationUsersByService.put(serviceName, delegationUsers); + configuredDelegationUsers.put(serviceName, delegationUsers); LOG.info("Delegation users for service '{}': {}", serviceName, delegationUsers); } } } - if (delegationUsersByService.isEmpty()) { + delegationUsersByService = mergeDelegationUsers(configuredDelegationUsers, Collections.emptyMap()); + + if (configuredDelegationUsers.isEmpty()) { LOG.warn("No delegation users configured"); + } + } + + void refreshDelegationUsers() { + RangerAdminRESTClient currentRangerAdminClient = rangerAdminClient; + + if (currentRangerAdminClient != null) { + try { + delegationUsersByService = mergeDelegationUsers(configuredDelegationUsers, currentRangerAdminClient.getServiceAccountUsers()); + } catch (Exception e) { + LOG.warn("Failed to refresh service account users from Ranger Admin", e); + } } else { - Set wildcardDelegationUsers = delegationUsersByService.get(WILDCARD_SERVICE_NAME); + LOG.warn("Failed to refresh service account users from Ranger admin: admin client not initialized"); + } + } + + static Map> mergeDelegationUsers(Map> configuredDelegationUsers, Map> serviceAccountUsersByServiceName) { + Map> ret = new HashMap<>(); + Set wildcardDelegationUsers = configuredDelegationUsers.get(WILDCARD_SERVICE_NAME); + + if (wildcardDelegationUsers == null) { + wildcardDelegationUsers = Collections.emptySet(); + } + + for (Map.Entry> entry : configuredDelegationUsers.entrySet()) { + Set delegationUsers = new HashSet<>(entry.getValue()); + + if (!WILDCARD_SERVICE_NAME.equals(entry.getKey())) { + delegationUsers.addAll(wildcardDelegationUsers); + } + + ret.put(entry.getKey(), delegationUsers); + } + + for (Map.Entry> entry : serviceAccountUsersByServiceName.entrySet()) { + String serviceName = entry.getKey(); + Set serviceAccountUsers = entry.getValue(); + + if (StringUtils.isNotBlank(serviceName) && serviceAccountUsers != null && !serviceAccountUsers.isEmpty()) { + Set delegationUsersForService = ret.get(serviceName); + Set delegationUsers = new HashSet<>(); + + if (delegationUsersForService != null) { + delegationUsers.addAll(delegationUsersForService); + } else if (!WILDCARD_SERVICE_NAME.equals(serviceName)) { + delegationUsers.addAll(wildcardDelegationUsers); + } + + delegationUsers.addAll(serviceAccountUsers); + ret.put(serviceName, delegationUsers); + } + } + + return ret; + } + + private void startServiceAccountUsersRefresh(Properties properties) { + if (!Boolean.parseBoolean(StringUtils.defaultString(properties.getProperty(PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED), "false"))) { + LOG.info("service account user refresh is not enabled. Property: {}", PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED); + return; + } + + LOG.info("Starting service account user refresher"); + + String rangerAdminUrl = properties.getProperty(PROP_POLICY_REST_URL); + + if (serviceAccountUsersRefreshTimer == null && StringUtils.isNotBlank(rangerAdminUrl)) { + try { + Configuration rangerAdminConfig = new Configuration(false); + RangerAdminRESTClient client = new RangerAdminRESTClient(); + + for (String key : properties.stringPropertyNames()) { + String value = properties.getProperty(key); + + if (value != null) { + rangerAdminConfig.set(key, value); + } + } + + client.init("ranger-pdp", "ranger-pdp", StringUtils.removeEnd(RangerAuthzConfig.PROP_PREFIX_DEFAULT, "."), rangerAdminConfig); + + rangerAdminClient = client; + + long pollIntervalMs = rangerAdminConfig.getLong(PROP_POLICY_POLL_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS); + Timer timer = new Timer("pdpServiceAccountUsersRefresh", true); + + refreshDelegationUsers(); + + timer.schedule(new TimerTask() { + @Override + public void run() { + refreshDelegationUsers(); + } + }, pollIntervalMs, pollIntervalMs); + + serviceAccountUsersRefreshTimer = timer; - // delegation users for WILDCARD_SERVICE_NAME are delegates for all services - if (wildcardDelegationUsers != null && !wildcardDelegationUsers.isEmpty()) { - delegationUsersByService.forEach((k, v) -> v.addAll(wildcardDelegationUsers)); + LOG.info("Scheduled service account user refresh every {} ms from {}", pollIntervalMs, rangerAdminUrl); + } catch (Exception e) { + LOG.warn("Failed to start service account user refresh", e); } } } diff --git a/pdp/src/main/resources/ranger-pdp-default.xml b/pdp/src/main/resources/ranger-pdp-default.xml index 6f3152f4157..88ac3a6ad68 100644 --- a/pdp/src/main/resources/ranger-pdp-default.xml +++ b/pdp/src/main/resources/ranger-pdp-default.xml @@ -230,12 +230,20 @@ Policy source implementation. Use RangerAdminRESTClient to download policies live from Ranger Admin. + + ranger.pdp.serviceaccount.refresher.enabled + false + When true and ranger.authz.default.policy.rest.url is set, periodically merge service.account.users from Ranger Admin into PDP delegation users. Override via Cloudera Manager safety valve. Poll credentials use ranger.authz.default.policy.rest.* (same as policy download). + + ranger.authz.default.policy.rest.url http://localhost:6080 URL of the Ranger Admin server. Comma-separated for HA. Example: http://ranger-admin-1:6080,http://ranger-admin-2:6080 + Also used for service-account user refresh when ranger.pdp.serviceaccount.refresher.enabled is true (rest client auth via ranger.authz.default.policy.rest.ssl.config.file and .client.username/.password). + The REST client identity must be listed in ranger.admin.service.account.info.viewers on Admin. @@ -268,7 +276,7 @@ ranger.authz.default.policy.pollIntervalMs 30000 - How often (ms) to poll Ranger Admin for updated policies and roles. + How often (ms) to poll Ranger Admin for updated policies and roles and to refresh service-account users when ranger.pdp.serviceaccount.refresher.enabled is true. diff --git a/pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java b/pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java new file mode 100644 index 00000000000..8b0f617bd98 --- /dev/null +++ b/pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.pdp.rest; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PdpServiceAccountRefreshBootstrapTest { + @Test + void startStopDelegateToRangerPdpREST() { + TrackingPdpRest rest = new TrackingPdpRest(); + PdpServiceAccountRefreshBootstrap bootstrap = new PdpServiceAccountRefreshBootstrap(rest); + + bootstrap.start(); + bootstrap.stop(); + + assertTrue(rest.startCalled); + assertTrue(rest.stopCalled); + } + + private static final class TrackingPdpRest extends RangerPdpREST { + boolean startCalled; + boolean stopCalled; + + @Override + public synchronized void startServiceAccountUsersRefresh() { + startCalled = true; + } + + @Override + public synchronized void stopServiceAccountUsersRefresh() { + stopCalled = true; + } + } +} diff --git a/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java b/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java index 1221af3bbdc..5832a58796f 100644 --- a/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java +++ b/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java @@ -19,12 +19,14 @@ package org.apache.ranger.pdp.rest; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authz.api.RangerAuthorizer; import org.apache.ranger.authz.api.RangerAuthzApiErrorCode; import org.apache.ranger.authz.api.RangerAuthzException; import org.apache.ranger.authz.model.RangerAccessContext; import org.apache.ranger.authz.model.RangerAuthzRequest; import org.apache.ranger.authz.model.RangerAuthzResult; +import org.apache.ranger.authz.model.RangerFilterResourcesRequest; import org.apache.ranger.authz.model.RangerMultiAuthzRequest; import org.apache.ranger.authz.model.RangerMultiAuthzResult; import org.apache.ranger.authz.model.RangerResourcePermissions; @@ -41,13 +43,18 @@ import java.lang.reflect.Field; import java.lang.reflect.Proxy; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Properties; +import java.util.Set; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_PDP_SERVICE_PREFIX; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SUFFIX_DELEGATION_USERS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; public class RangerPdpRESTTest { private static final String CALLER_DELEGATION_ALLOWED = "alice"; @@ -202,6 +209,94 @@ public void testGetResourcePermissionsReturnsBadRequestOnAuthzException() throws assertEquals(1L, stats.getTotalAuthzBadRequest()); } + @Test + public void testMergeDelegationUsersUnionsServiceAccountUsersAndWildcard() { + Map> configuredDelegationUsers = new HashMap<>(); + + configuredDelegationUsers.put("*", new HashSet<>(Collections.singleton("wild"))); + configuredDelegationUsers.put("svc1", new HashSet<>(Collections.singleton(CALLER_DELEGATION_ALLOWED))); + + Map> serviceAccountUsersByServiceName = new HashMap<>(); + + serviceAccountUsersByServiceName.put("svc1", Collections.singleton("svc-user")); + serviceAccountUsersByServiceName.put("svc2", Collections.singleton("only-dynamic")); + + Map> mergedDelegationUsers = RangerPdpREST.mergeDelegationUsers(configuredDelegationUsers, serviceAccountUsersByServiceName); + + assertEquals(new HashSet<>(Arrays.asList(CALLER_DELEGATION_ALLOWED, "wild", "svc-user")), mergedDelegationUsers.get("svc1")); + assertEquals(new HashSet<>(Arrays.asList("wild", "only-dynamic")), mergedDelegationUsers.get("svc2")); + assertEquals(Collections.singleton("wild"), mergedDelegationUsers.get("*")); + } + + @Test + public void testRefreshKeepsDelegationUsersWhenAdminCallFails() throws Exception { + RangerPdpREST rest = createRest(new TestAuthorizer(), new RangerPdpStats()); + Map> before = delegationUsers(rest); + + setField(rest, "rangerAdminClient", new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() throws Exception { + throw new Exception("admin down"); + } + }); + + rest.refreshDelegationUsers(); + + assertSame(before, delegationUsers(rest)); + } + + @Test + public void testRefreshAddsServiceAccountUsers() throws Exception { + RangerPdpStats stats = new RangerPdpStats(); + RangerPdpREST rest = createRest(new TestAuthorizer(new RangerAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW)), stats); + RangerAuthzRequest request = new RangerAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1); + + setField(rest, "rangerAdminClient", new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.singletonMap("svc1", Collections.singleton("svc-user")); + } + }); + + rest.refreshDelegationUsers(); + + Response response = rest.authorize(request, httpRequest("svc-user", stats)); + + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + } + + @Test + public void testServiceAccountUserCanCallAllPdpApis() throws Exception { + RangerPdpStats stats = new RangerPdpStats(); + TestAuthorizer authorizer = new TestAuthorizer( + new RangerAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW), + new RangerMultiAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW), + new RangerResourcePermissions(), null); + RangerPdpREST rest = createRest(authorizer, stats); + String dynamicCaller = "svc-user"; + + setField(rest, "rangerAdminClient", new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.singletonMap("svc1", Collections.singleton(dynamicCaller)); + } + }); + rest.refreshDelegationUsers(); + + assertEquals(Response.Status.OK.getStatusCode(), rest.authorize( + new RangerAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1), + httpRequest(dynamicCaller, stats)).getStatus()); + assertEquals(Response.Status.OK.getStatusCode(), rest.authorizeMulti( + new RangerMultiAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1), + httpRequest(dynamicCaller, stats)).getStatus()); + assertEquals(Response.Status.OK.getStatusCode(), rest.getResourcePermissions( + new RangerResourcePermissionsRequest("req-1", null, CONTEXT_SERVICE_SVC1), + httpRequest(dynamicCaller, stats)).getStatus()); + assertEquals(Response.Status.OK.getStatusCode(), rest.filterResources( + new RangerFilterResourcesRequest("user1", Arrays.asList("table:default/t1"), "_any", "hive", "svc1"), + httpRequest(dynamicCaller, stats)).getStatus()); + } + private static RangerPdpREST createRest(TestAuthorizer authorizer, RangerPdpStats stats) throws Exception { RangerPdpREST rest = new RangerPdpREST(); @@ -213,6 +308,15 @@ private static RangerPdpREST createRest(TestAuthorizer authorizer, RangerPdpStat return rest; } + @SuppressWarnings("unchecked") + private static Map> delegationUsers(RangerPdpREST rest) throws Exception { + Field field = RangerPdpREST.class.getDeclaredField("delegationUsersByService"); + + field.setAccessible(true); + + return (Map>) field.get(rest); + } + private static HttpServletRequest httpRequest(String user, RangerPdpStats stats) { Map attrs = new HashMap<>(); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 5bd9d21d2f7..116b33092a9 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -1030,7 +1030,8 @@ public Boolean hasAccess(XXDBBase xxDbBase, RangerBaseModelObject baseModel) { } if (baseModel instanceof RangerServiceHeaderInfo) { - return hasAccessToRangerServiceHeaderInfo((RangerServiceHeaderInfo) baseModel, isKeyAdmin, isSysAdmin, isAuditor, isAuditorKeyAdmin, isUser); + return hasServiceAccountInfoAccess() || + hasAccessToRangerServiceHeaderInfo((RangerServiceHeaderInfo) baseModel, isKeyAdmin, isSysAdmin, isAuditor, isAuditorKeyAdmin, isUser); } return false; diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index 09326c6514c..b38e6f98750 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -437,7 +437,7 @@ public List searchServices(@Context HttpServletRequest request) { @GET @Path("/api/service-headers") @Produces("application/json") - @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_SERVICE_HEADERS + "\")") + @PreAuthorize("@rangerBizUtil.hasServiceAccountInfoAccess() || @rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_SERVICE_HEADERS + "\")") public List getServiceHeaders(@Context HttpServletRequest request) { return serviceREST.getServiceHeaders(request); } diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 4f7bc87acf8..2a6e74eff78 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -212,6 +212,14 @@ roles. In many cases, this kind of downloading behavior does not need to be protected. + + ranger.admin.service.account.info.viewers + ranger-pdp,ranger-audit-ingestor,admin,rangerauditserver + + Comma-separated logins (and Kerberos short names) allowed to receive service.account.users and service.account.groups on GET /public/v2/api/service-headers. + Must include identities used by ranger.pdp.serviceaccount.refresher.enabled / ranger.audit.ingestor.serviceaccount.refresher.enabled (ranger.authz.default.policy.rest.* and ranger.audit.ingestor.policy.rest.* clients). Safety-valve append on this list. + +