From b2080152a17f9d2e35c42a17667ae69324fe4b40 Mon Sep 17 00:00:00 2001 From: akgupta Date: Mon, 16 Nov 2026 21:08:26 +0530 Subject: [PATCH 1/4] RANGER-5869:Ranger PDP and audit ingestor services to dynamically obtain authorized callers --- .../admin/client/RangerAdminRESTClient.java | 65 +++++++- .../ranger/plugin/util/RangerRESTUtils.java | 1 + .../util/ServiceAccountUsersProvider.java | 99 ++++++++++++ .../util/ServiceAccountUsersRefresher.java | 146 ++++++++++++++++++ .../client/TestRangerAdminRESTClient.java | 44 ++++++ .../util/TestServiceAccountUsersProvider.java | 86 +++++++++++ .../audit/server/AuditServerConstants.java | 1 + .../apache/ranger/audit/rest/AuditREST.java | 8 + .../AuditServiceAccountUsersConfig.java | 46 ++++++ .../conf/ranger-audit-ingestor-site.xml | 18 +++ .../ranger/audit/rest/AuditRESTTest.java | 113 ++++++++++++++ pdp/conf.dist/ranger-pdp-site.xml | 12 ++ .../apache/ranger/pdp/RangerPdpServer.java | 27 +++- .../ranger/pdp/config/RangerPdpConfig.java | 25 ++- .../ranger/pdp/config/RangerPdpConstants.java | 2 + .../ranger/pdp/rest/RangerPdpApplication.java | 18 ++- .../apache/ranger/pdp/rest/RangerPdpREST.java | 21 +-- pdp/src/main/resources/ranger-pdp-default.xml | 12 ++ .../ranger/pdp/rest/RangerPdpRESTTest.java | 42 +++++ .../org/apache/ranger/biz/RangerBizUtil.java | 3 +- .../org/apache/ranger/rest/PublicAPIsv2.java | 2 +- .../conf.dist/ranger-admin-default-site.xml | 8 + 22 files changed, 775 insertions(+), 24 deletions(-) create mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java create mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java create mode 100644 agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index dfd295bbce0..bd2d2d05c72 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -28,6 +28,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.model.RangerRole; +import org.apache.ranger.plugin.model.RangerServiceHeaderInfo; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; import org.apache.ranger.plugin.util.JsonUtilsV2; @@ -51,15 +52,18 @@ import java.io.UnsupportedEncodingException; import java.security.PrivilegedExceptionAction; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; public class RangerAdminRESTClient extends AbstractRangerAdminClient { private static final Logger LOG = LoggerFactory.getLogger(RangerAdminRESTClient.class); - private static final TypeReference> TYPE_LIST_STRING = new TypeReference>() {}; + private static final TypeReference> TYPE_LIST_STRING = new TypeReference>() {}; + private static final TypeReference> TYPE_LIST_SERVICE_HEADER = new TypeReference>() {}; private final String pluginCapabilities = Long.toHexString(new RangerPluginCapability().getPluginCapabilities()); private final RangerRESTUtils restUtils = new RangerRESTUtils(); @@ -116,7 +120,7 @@ public void init(String serviceName, String appId, String propertyPrefix, Config url = url.substring(0, url.length() - 1); } - init(url, sslConfigFileName, restClientConnTimeOutMs, restClientReadTimeOutMs, restClientMaxRetryAttempts, restClientRetryIntervalMs, config); + init(url, sslConfigFileName, restClientConnTimeOutMs, restClientReadTimeOutMs, restClientMaxRetryAttempts, restClientRetryIntervalMs, config, propertyPrefix); try { this.serviceNameUrlParam = URLEncoderUtil.encodeURIParam(serviceName); @@ -126,6 +130,59 @@ public void init(String serviceName, String appId, String propertyPrefix, Config } } + public Map> getServiceAccountUsers() throws Exception { + LOG.debug("==> RangerAdminRESTClient.getServiceAccountUsers()"); + + final boolean isSecureMode = isAuthenticationEnabled(); + final Cookie sessionId = this.sessionId; + final Map queryParams = Collections.emptyMap(); + final Response response; + + if (isSecureMode) { + response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction) () -> + restClient.get(RangerRESTUtils.REST_URL_SERVICE_HEADERS, queryParams, sessionId)); + } else { + response = restClient.get(RangerRESTUtils.REST_URL_SERVICE_HEADERS, queryParams, sessionId); + } + + checkAndResetSessionCookie(response); + + Map> ret = Collections.emptyMap(); + + if (response == null) { + throw new Exception("Unknown error while retrieving service-account users"); + } else if (response.getStatus() == HttpStatus.SC_OK) { + List serviceHeaders = JsonUtilsV2.jsonToObj(response.readEntity(String.class), + TYPE_LIST_SERVICE_HEADER); + + if (serviceHeaders != null && !serviceHeaders.isEmpty()) { + Map> serviceAccountUsers = new HashMap<>(); + + for (RangerServiceHeaderInfo serviceHeader : serviceHeaders) { + if (serviceHeader != null && Boolean.TRUE.equals(serviceHeader.getIsEnabled()) && + !StringUtil.isEmpty(serviceHeader.getName()) && serviceHeader.getServiceAccountUsers() != null && + !serviceHeader.getServiceAccountUsers().isEmpty()) { + serviceAccountUsers.put(serviceHeader.getName(), serviceHeader.getServiceAccountUsers()); + } + } + + ret = serviceAccountUsers; + } + } else { + LOG.error("getServiceAccountUsers() failed: HTTP status={}, isSecure={}", response.getStatus(), isSecureMode); + + if (response.getStatus() == HttpStatus.SC_UNAUTHORIZED || response.getStatus() == HttpStatus.SC_FORBIDDEN) { + throw new AccessControlException(); + } + + throw new Exception("HTTP " + response.getStatus() + " while retrieving service-account users"); + } + + LOG.debug("<== RangerAdminRESTClient.getServiceAccountUsers(): serviceCount={}", ret.size()); + + return ret; + } + @Override public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated({}, {})", lastKnownVersion, lastActivationTimeInMillis); @@ -1029,10 +1086,10 @@ public void setTokenSupplier(Supplier tokenSupplier) { } } - private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs, int restClientReadTimeOutMs, int restClientMaxRetryAttempts, int restClientRetryIntervalMs, Configuration config) { + private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs, int restClientReadTimeOutMs, int restClientMaxRetryAttempts, int restClientRetryIntervalMs, Configuration config, String propertyPrefix) { LOG.debug("==> RangerAdminRESTClient.init({}, {})", url, sslConfigFileName); - restClient = new RangerRESTClient(url, sslConfigFileName, config); + restClient = new RangerRESTClient(url, sslConfigFileName, config, propertyPrefix); if (tokenSupplier != null) { restClient.setTokenSupplier(tokenSupplier); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java index bcff32a2e6b..7aaff8c6df0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java @@ -43,6 +43,7 @@ public class RangerRESTUtils { public static final String REST_URL_SERVICE_GET_ROLE_INFO = "/service/public/v2/api/roles/name/"; public static final String REST_URL_SERVICE_GRANT_ROLE = "/service/public/v2/api/roles/grant/"; public static final String REST_URL_SERVICE_REVOKE_ROLE = "/service/public/v2/api/roles/revoke/"; + public static final String REST_URL_SERVICE_HEADERS = "/service/public/v2/api/service-headers"; public static final String REST_URL_SERVICE_SERCURE_GET_USER_GROUP_ROLES = "/service/roles/secure/download/"; public static final String REST_URL_SERVICE_GET_USER_GROUP_ROLES = "/service/roles/download/"; public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download/"; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java new file mode 100644 index 00000000000..e0531a1290e --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.admin.client.RangerAdminRESTClient; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class ServiceAccountUsersProvider { + private static final String CONFIG_PREFIX = "ranger.serviceaccount.refresher"; + private static final String POLLING_INTERVAL_PROPERTY = CONFIG_PREFIX + ".pollIntervalMs"; + + private final ServiceAccountUsersRefresher refresher; + + private volatile Map> serviceAccountUsers = Collections.emptyMap(); + + public ServiceAccountUsersProvider(boolean serviceAccountRefreshEnabled, Configuration config) { + refresher = serviceAccountRefreshEnabled ? createRefresher(requireNonNull(config, "config")) : null; + } + + public ServiceAccountUsersProvider(boolean serviceAccountRefreshEnabled, Properties properties) { + this(serviceAccountRefreshEnabled, serviceAccountRefreshEnabled ? toConfiguration(properties) : null); + } + + public void start() { + if (refresher != null) { + refresher.startRefresher(); + } + } + + public void stop() { + if (refresher != null) { + refresher.stopRefresher(); + } + } + + public boolean isServiceAccountUser(String serviceName, String userName) { + Set users = serviceAccountUsers.get(serviceName); + + return users != null && users.contains(userName); + } + + private ServiceAccountUsersRefresher createRefresher(Configuration config) { + RangerAdminRESTClient rangerAdmin = new RangerAdminRESTClient(); + + rangerAdmin.init("service-account-users", "service-account-users-provider", CONFIG_PREFIX, config); + + return new ServiceAccountUsersRefresher(rangerAdmin::getServiceAccountUsers, this::setServiceAccountUsers, + config.getLong(POLLING_INTERVAL_PROPERTY, -1L)); + } + + private static Configuration toConfiguration(Properties properties) { + Configuration ret = new Configuration(false); + + Properties source = requireNonNull(properties, "properties"); + + for (String name : source.stringPropertyNames()) { + ret.set(name, source.getProperty(name)); + } + + return ret; + } + + private void setServiceAccountUsers(Map> serviceAccountUsers) { + Map> snapshot = new HashMap<>(); + + serviceAccountUsers.forEach((serviceName, users) -> { + if (serviceName != null && users != null && !users.isEmpty()) { + snapshot.put(serviceName, Collections.unmodifiableSet(new HashSet<>(users))); + } + }); + + this.serviceAccountUsers = Collections.unmodifiableMap(snapshot); + } +} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java new file mode 100644 index 00000000000..889db1a96f7 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Set; +import java.util.Timer; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Consumer; + +import static java.util.Objects.requireNonNull; + +final class ServiceAccountUsersRefresher extends Thread { + private static final Logger LOG = LoggerFactory.getLogger(ServiceAccountUsersRefresher.class); + + private final Callable>> serviceAccountUsersLoader; + private final Consumer>> receiver; + private final long pollingIntervalMs; + private final BlockingQueue downloadQueue = new LinkedBlockingQueue<>(); + private Timer downloadTimer; + + ServiceAccountUsersRefresher(Callable>> serviceAccountUsersLoader, Consumer>> receiver, long pollingIntervalMs) { + if (pollingIntervalMs <= 0) { + throw new IllegalArgumentException("pollingIntervalMs must be greater than zero"); + } + + this.serviceAccountUsersLoader = requireNonNull(serviceAccountUsersLoader, "serviceAccountUsersLoader"); + this.receiver = requireNonNull(receiver, "receiver"); + this.pollingIntervalMs = pollingIntervalMs; + + setName("ServiceAccountUsersRefresher-" + getId()); + setDaemon(true); + } + + void startRefresher() { + refresh(); + + try { + start(); + + downloadTimer = new Timer("serviceAccountUsersDownloadTimer", true); + downloadTimer.schedule(new DownloaderTask(downloadQueue), pollingIntervalMs, pollingIntervalMs); + + LOG.info("Started service-account user refresher with polling interval {} ms", pollingIntervalMs); + } catch (RuntimeException excp) { + LOG.error("Failed to start service-account user refresher", excp); + + stopRefresher(); + + throw excp; + } + } + + void stopRefresher() { + boolean wasStarted = downloadTimer != null || isAlive(); + Timer downloadTimer = this.downloadTimer; + + this.downloadTimer = null; + + if (downloadTimer != null) { + downloadTimer.cancel(); + } + + if (isAlive()) { + interrupt(); + + boolean interrupted = false; + boolean joined = false; + + while (!joined) { + try { + join(); + + joined = true; + } catch (InterruptedException excp) { + LOG.warn("Interrupted while waiting for service-account user refresher to stop", excp); + + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + if (wasStarted) { + LOG.info("Stopped service-account user refresher"); + } + } + + @Override + public void run() { + while (true) { + DownloadTrigger trigger = null; + + try { + trigger = downloadQueue.take(); + + refresh(); + } catch (InterruptedException excp) { + LOG.info("Service-account user refresher interrupted; exiting"); + + break; + } finally { + if (trigger != null) { + trigger.signalCompletion(); + } + } + } + } + + void refresh() { + try { + Map> serviceAccountUsers = requireNonNull(serviceAccountUsersLoader.call(), + "Ranger Admin returned null service-account users"); + + receiver.accept(serviceAccountUsers); + + LOG.debug("Refreshed service-account users for {} services", serviceAccountUsers.size()); + } catch (Exception excp) { + LOG.warn("Failed to refresh service-account users; retaining the last successful snapshot", excp); + } + } +} diff --git a/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java b/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java index 1ffd1731b95..1f63e562ab4 100644 --- a/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java +++ b/agents-common/src/test/java/org/apache/ranger/admin/client/TestRangerAdminRESTClient.java @@ -29,6 +29,7 @@ import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.plugin.util.RangerRESTUtils; import org.apache.ranger.plugin.util.RangerRoles; import org.apache.ranger.plugin.util.RangerServiceNotFoundException; import org.apache.ranger.plugin.util.ServiceGdsInfo; @@ -51,8 +52,10 @@ import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; /** * @generated by Cursor @@ -1146,4 +1149,45 @@ public void test42_secureEndpointAccessDisabled_withoutKerberos_usesNonSecureUrl misc.verify(() -> MiscUtil.executePrivilegedAction(Mockito.>any()), Mockito.never()); } } + + @Test + public void test43_getServiceAccountUsers() throws Exception { + RangerAdminRESTClient client = new RangerAdminRESTClient(); + Configuration cfg = new Configuration(false); + cfg.set("p.policy.rest.url", "http://localhost:6080"); + cfg.setBoolean("p.forceSecureEndpointAccess", false); + client.init("svc", "app", "p", cfg); + RangerRESTClient rest = Mockito.mock(RangerRESTClient.class); + setPrivateField(client, "restClient", rest); + + Response response = mockResponse(200, + "[{\"name\":\"svc1\",\"isEnabled\":true,\"serviceAccountUsers\":[\"alice\"]}," + + "{\"name\":\"svc2\",\"isEnabled\":false,\"serviceAccountUsers\":[\"bob\"]}," + + "{\"name\":\"svc3\",\"isEnabled\":true}]", + Collections.emptyList()); + Mockito.when(rest.get(Mockito.anyString(), Mockito.anyMap(), Mockito.any())).thenReturn(response); + + Map> users = client.getServiceAccountUsers(); + + Assertions.assertEquals(Collections.singleton("alice"), users.get("svc1")); + Assertions.assertEquals(1, users.size()); + Mockito.verify(rest).get(Mockito.eq(RangerRESTUtils.REST_URL_SERVICE_HEADERS), Mockito.anyMap(), Mockito.any()); + } + + @Test + public void test44_getServiceAccountUsers_forbidden() throws Exception { + RangerAdminRESTClient client = new RangerAdminRESTClient(); + Configuration cfg = new Configuration(false); + cfg.set("p.policy.rest.url", "http://localhost:6080"); + cfg.setBoolean("p.forceSecureEndpointAccess", false); + client.init("svc", "app", "p", cfg); + RangerRESTClient rest = Mockito.mock(RangerRESTClient.class); + setPrivateField(client, "restClient", rest); + Response response = Mockito.mock(Response.class); + Mockito.when(response.getStatus()).thenReturn(403); + Mockito.when(rest.get(Mockito.anyString(), Mockito.anyMap(), Mockito.any())) + .thenReturn(response); + + Assertions.assertThrows(AccessControlException.class, client::getServiceAccountUsers); + } } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java b/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java new file mode 100644 index 00000000000..441f22ceb39 --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestServiceAccountUsersProvider { + @Test + public void testProviderLoadsDefensiveSnapshot() throws Exception { + Set users = new HashSet<>(Collections.singleton("alice")); + Map> source = new HashMap<>(); + ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, (Configuration) null); + Method setUsers = ServiceAccountUsersProvider.class.getDeclaredMethod("setServiceAccountUsers", Map.class); + + source.put("svc", users); + + setUsers.setAccessible(true); + setUsers.invoke(provider, source); + users.add("bob"); + + assertTrue(provider.isServiceAccountUser("svc", "alice")); + assertFalse(provider.isServiceAccountUser("svc", "bob")); + + setUsers.invoke(provider, Collections.emptyMap()); + assertFalse(provider.isServiceAccountUser("svc", "alice")); + } + + @Test + public void testRefresherReplacesSnapshotAndRetainsItOnFailure() { + AtomicInteger calls = new AtomicInteger(); + AtomicReference>> received = new AtomicReference<>(); + ServiceAccountUsersRefresher refresher = new ServiceAccountUsersRefresher(() -> { + if (calls.getAndIncrement() == 0) { + return Collections.singletonMap("svc", Collections.singleton("alice")); + } + + throw new Exception("expected"); + }, received::set, 60_000L); + + refresher.refresh(); + Map> snapshot = received.get(); + refresher.refresh(); + + assertSame(snapshot, received.get()); + } + + @Test + public void testDisabledProviderIsNoOp() { + ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, (Configuration) null); + + assertDoesNotThrow(provider::start); + assertFalse(provider.isServiceAccountUser("svc", "alice")); + assertDoesNotThrow(provider::stop); + } +} diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index bcd3c955f57..d79202e4cac 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -27,6 +27,7 @@ private AuditServerConstants() {} public static final String PROP_PREFIX_AUDIT_SERVER_SERVICE = PROP_PREFIX_AUDIT_SERVER + "service."; public static final String PROP_AUTH_TO_LOCAL = PROP_PREFIX_AUDIT_SERVER + "auth.to.local"; public static final String PROP_SUFFIX_ALLOWED_USERS = ".allowed.users"; + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = "ranger.serviceaccount.refresher.enabled"; public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "service.kerberos.principal"; public static final String PROP_AUDIT_SERVICE_KEYTAB = "service.kerberos.keytab"; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index 37c2fce39af..af334c74970 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -26,6 +26,7 @@ import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -68,6 +69,9 @@ public class AuditREST { @Autowired AuditDestinationMgr auditDestinationMgr; + @Autowired + ServiceAccountUsersProvider serviceAccountUsersProvider; + /** * Health check endpoint */ @@ -333,6 +337,10 @@ private boolean isAllowedServiceUser(String serviceName, String userName) { Set allowedUsers = allowedServiceUsers.get(serviceName); ret = allowedUsers != null && allowedUsers.contains(userName); + + if (!ret && serviceAccountUsersProvider != null) { + ret = serviceAccountUsersProvider.isServiceAccountUser(serviceName, userName); + } } else { ret = false; } diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java new file mode 100644 index 00000000000..6a12ec9d36a --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.server; + +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AuditServiceAccountUsersConfig { + private static final Logger LOG = LoggerFactory.getLogger(AuditServiceAccountUsersConfig.class); + + @Bean(initMethod = "start", destroyMethod = "stop") + public ServiceAccountUsersProvider serviceAccountUsersProvider() { + AuditServerConfig config = AuditServerConfig.getInstance(); + boolean enabled = config.getBoolean(AuditServerConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, true); + + try { + LOG.info("Service-account user refresher is configured as {}", enabled ? "enabled" : "disabled"); + + return new ServiceAccountUsersProvider(enabled, config); + } catch (RuntimeException excp) { + LOG.error("Failed to initialize service-account user refresher; continuing with configured allowed users", excp); + + return new ServiceAccountUsersProvider(false, config); + } + } +} diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index d3bf921cff0..3b8a681d424 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -50,6 +50,24 @@ 7182 + + ranger.serviceaccount.refresher.enabled + true + Enable periodic retrieval of service-account users from Ranger Admin. + + + + ranger.serviceaccount.refresher.policy.rest.url + http://localhost:6080 + Ranger Admin base URL used to retrieve service-account users. + + + + ranger.serviceaccount.refresher.pollIntervalMs + 30000 + How often (ms) to refresh service-account users from Ranger Admin. + + ranger.audit.ingestor.https.attrib.ssl.enabled false diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java new file mode 100644 index 00000000000..cdf8467e711 --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.rest; + +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.producer.AuditDestinationMgr; +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; +import org.junit.jupiter.api.Test; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Response; + +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.security.Principal; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class AuditRESTTest { + private static final String SERVICE = "dynamic-service"; + private static final String USER = "dynamic-user"; + + @Test + void testServiceAccountUserCanPostAudits() throws Exception { + TestDestination destination = new TestDestination(); + List audits = Collections.singletonList(new AuthzAuditEvent()); + AuditREST rest = createRest(destination, provider(true)); + + Response response = rest.logAccessAudit(SERVICE, "app", audits, request(USER)); + + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + assertTrue(destination.called); + } + + @Test + void testUnknownUserCannotPostAudits() throws Exception { + TestDestination destination = new TestDestination(); + List audits = Collections.singletonList(new AuthzAuditEvent()); + AuditREST rest = createRest(destination, provider(false)); + + Response response = rest.logAccessAudit(SERVICE, "app", audits, request(USER)); + + assertEquals(Response.Status.FORBIDDEN.getStatusCode(), response.getStatus()); + assertFalse(destination.called); + } + + private static AuditREST createRest(AuditDestinationMgr destination, ServiceAccountUsersProvider provider) { + AuditREST ret = new AuditREST(); + + ret.auditDestinationMgr = destination; + ret.serviceAccountUsersProvider = provider; + + return ret; + } + + private static HttpServletRequest request(String user) { + Principal principal = () -> user; + + return (HttpServletRequest) Proxy.newProxyInstance( + HttpServletRequest.class.getClassLoader(), + new Class[] {HttpServletRequest.class}, + (proxy, method, args) -> "getUserPrincipal".equals(method.getName()) ? principal : null); + } + + private static ServiceAccountUsersProvider provider(boolean allowed) throws Exception { + ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, new Properties()); + Map> users = allowed + ? Collections.singletonMap(SERVICE, Collections.singleton(USER)) + : Collections.emptyMap(); + Field field = ServiceAccountUsersProvider.class.getDeclaredField("serviceAccountUsers"); + + field.setAccessible(true); + field.set(provider, users); + + return provider; + } + + private static class TestDestination extends AuditDestinationMgr { + private boolean called; + + @Override + public boolean logBatch(Collection events, String appId) { + called = true; + + return true; + } + } +} diff --git a/pdp/conf.dist/ranger-pdp-site.xml b/pdp/conf.dist/ranger-pdp-site.xml index 6f3152f4157..f1c7334c3df 100644 --- a/pdp/conf.dist/ranger-pdp-site.xml +++ b/pdp/conf.dist/ranger-pdp-site.xml @@ -271,6 +271,18 @@ How often (ms) to poll Ranger Admin for updated policies and roles. + + ranger.serviceaccount.refresher.enabled + true + Enable periodic retrieval of service-account users from Ranger Admin. + + + + ranger.serviceaccount.refresher.pollIntervalMs + 30000 + How often (ms) to refresh service-account users from Ranger Admin. + + ranger.authz.default.policy.cache.dir /var/ranger/cache/pdp diff --git a/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java b/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java index 2f7ba67b199..7ee5421eb77 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java @@ -33,6 +33,7 @@ import org.apache.ranger.pdp.rest.RangerPdpApplication; import org.apache.ranger.pdp.security.RangerPdpAuthNFilter; import org.apache.ranger.pdp.security.RangerPdpRequestContextFilter; +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.apache.tomcat.util.descriptor.web.FilterDef; import org.apache.tomcat.util.descriptor.web.FilterMap; import org.glassfish.jersey.servlet.ServletContainer; @@ -65,6 +66,7 @@ public class RangerPdpServer { private final RangerPdpStats runtimeStats = new RangerPdpStats(); private Tomcat tomcat; private RangerAuthorizer authorizer; + private ServiceAccountUsersProvider serviceAccountUsersProvider; public RangerPdpServer() { this.config = new RangerPdpConfig(); @@ -78,6 +80,7 @@ public void start() throws Exception { LOG.info("Starting Ranger PDP server"); initAuthorizer(); + initServiceAccountUsersProvider(); startTomcat(); } @@ -107,6 +110,10 @@ public void stop() { LOG.warn("Error closing authorizer", e); } } + + if (serviceAccountUsersProvider != null) { + serviceAccountUsersProvider.stop(); + } } private void initAuthorizer() throws RangerAuthzException { @@ -121,6 +128,24 @@ private void initAuthorizer() throws RangerAuthzException { LOG.info("RangerEmbeddedAuthorizer initialised"); } + private void initServiceAccountUsersProvider() { + boolean enabled = config.isServiceAccountRefresherEnabled(); + ServiceAccountUsersProvider provider; + + try { + provider = new ServiceAccountUsersProvider(enabled, config.getServiceAccountRefresherProperties()); + provider.start(); + + LOG.info("Service-account user refresher is {}", enabled ? "enabled" : "disabled"); + } catch (RuntimeException excp) { + LOG.error("Failed to initialize service-account user refresher; continuing with configured delegation users", excp); + + provider = new ServiceAccountUsersProvider(false, config.getAuthzProperties()); + } + + serviceAccountUsersProvider = provider; + } + private void startTomcat() throws Exception { tomcat = new Tomcat(); @@ -255,7 +280,7 @@ private void addAuthFilter(Context ctx) { * that it can be injected into {@link org.apache.ranger.pdp.rest.RangerPdpREST}. */ private void addJerseyServlet(Context ctx) { - RangerPdpApplication jerseyApp = new RangerPdpApplication(authorizer, config); + RangerPdpApplication jerseyApp = new RangerPdpApplication(authorizer, config, serviceAccountUsersProvider); Servlet jerseyServlet = new ServletContainer(jerseyApp); Tomcat.addServlet(ctx, "jerseyServlet", jerseyServlet).setLoadOnStartup(1); diff --git a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java index d541d426888..b9c9ba428d8 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java @@ -52,6 +52,8 @@ public class RangerPdpConfig { private static final String DEFAULT_CONFIG_FILE = "ranger-pdp-default.xml"; private static final String SITE_CONFIG_FILE = "ranger-pdp-site.xml"; + private static final String DEFAULT_POLICY_REST_PREFIX = "ranger.authz.default.policy.rest."; + private static final String SERVICE_ACCOUNT_POLICY_REST_PREFIX = RangerPdpConstants.PROP_SERVICE_ACCOUNT_REFRESHER_PREFIX + "policy.rest."; private final Properties props = new Properties(); @@ -192,6 +194,26 @@ public Properties getAuthzProperties() { return new Properties(props); } + public Properties getServiceAccountRefresherProperties() { + Properties ret = getAuthzProperties(); + + for (String name : props.stringPropertyNames()) { + if (name.startsWith(DEFAULT_POLICY_REST_PREFIX)) { + String targetName = SERVICE_ACCOUNT_POLICY_REST_PREFIX + name.substring(DEFAULT_POLICY_REST_PREFIX.length()); + + if (ret.getProperty(targetName) == null) { + ret.setProperty(targetName, props.getProperty(name)); + } + } + } + + return ret; + } + + public boolean isServiceAccountRefresherEnabled() { + return getBoolean(RangerPdpConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, true); + } + public String get(String key, String defaultValue) { String val = props.getProperty(key); @@ -273,7 +295,8 @@ private void parseHadoopXml(InputStream in, String source) { */ private void applySystemPropertyOverrides() { for (String key : System.getProperties().stringPropertyNames()) { - if (key.startsWith(RangerPdpConstants.PROP_PDP_PREFIX) || key.startsWith(RangerPdpConstants.PROP_AUTHZ_PREFIX)) { + if (key.startsWith(RangerPdpConstants.PROP_PDP_PREFIX) || + key.startsWith(RangerPdpConstants.PROP_AUTHZ_PREFIX)) { props.setProperty(key, System.getProperty(key)); } } diff --git a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java index a0c40f29020..705d7253f45 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java @@ -87,6 +87,8 @@ private RangerPdpConstants() { public static final String PROP_AUTHZ_PREFIX = "ranger.authz."; public static final String PROP_PDP_PREFIX = "ranger.pdp."; public static final String PROP_PDP_SERVICE_PREFIX = PROP_PDP_PREFIX + "service."; + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_PREFIX = "ranger.serviceaccount.refresher."; + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_SERVICE_ACCOUNT_REFRESHER_PREFIX + "enabled"; // delegation users public static final String PROP_SUFFIX_DELEGATION_USERS = ".delegation.users"; diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java index 42371b452d0..64967d58381 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java @@ -21,6 +21,7 @@ import org.apache.ranger.authz.api.RangerAuthorizer; import org.apache.ranger.pdp.config.RangerPdpConfig; +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.server.ResourceConfig; @@ -38,25 +39,28 @@ * */ public class RangerPdpApplication extends ResourceConfig { - public RangerPdpApplication(RangerAuthorizer authorizer, RangerPdpConfig config) { + public RangerPdpApplication(RangerAuthorizer authorizer, RangerPdpConfig config, ServiceAccountUsersProvider serviceAccountUsersProvider) { register(JacksonFeature.class); - register(new AuthorizerBinder(authorizer, config)); + register(new AuthorizerBinder(authorizer, config, serviceAccountUsersProvider)); register(RangerPdpREST.class); } private static class AuthorizerBinder extends AbstractBinder { - private final RangerAuthorizer authorizer; - private final RangerPdpConfig config; + private final RangerAuthorizer authorizer; + private final RangerPdpConfig config; + private final ServiceAccountUsersProvider serviceAccountUsersProvider; - AuthorizerBinder(RangerAuthorizer authorizer, RangerPdpConfig config) { - this.authorizer = authorizer; - this.config = config; + AuthorizerBinder(RangerAuthorizer authorizer, RangerPdpConfig config, ServiceAccountUsersProvider serviceAccountUsersProvider) { + this.authorizer = authorizer; + this.config = config; + this.serviceAccountUsersProvider = serviceAccountUsersProvider; } @Override protected void configure() { bind(authorizer).to(RangerAuthorizer.class); bind(config).to(RangerPdpConfig.class); + bind(serviceAccountUsersProvider).to(ServiceAccountUsersProvider.class); } } } diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java index 99b23e2cad4..bbb8e464bd6 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java @@ -39,6 +39,7 @@ import org.apache.ranger.pdp.config.RangerPdpConfig; import org.apache.ranger.pdp.config.RangerPdpConstants; import org.apache.ranger.pdp.model.ErrorResponse; +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,6 +110,9 @@ public class RangerPdpREST { @Inject private RangerPdpConfig config; + @Inject + private ServiceAccountUsersProvider serviceAccountUsersProvider; + @Context private ServletContext servletContext; @@ -465,14 +469,9 @@ private void recordRequestMetrics(Response ret, long startNanos, HttpServletRequ } private boolean isDelegationUserForService(String serviceName, String userName) { - final boolean ret; - Map> delegationUsersByService = this.delegationUsersByService; + boolean ret = false; - if (delegationUsersByService.isEmpty()) { - ret = false; - } else if (StringUtils.isBlank(serviceName) || StringUtils.isBlank(userName)) { - ret = false; - } else { + if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) { Set delegationUsers = delegationUsersByService.get(serviceName); if (delegationUsers == null) { @@ -480,9 +479,13 @@ private boolean isDelegationUserForService(String serviceName, String userName) } ret = delegationUsers != null && delegationUsers.contains(userName); + + if (!ret && serviceAccountUsersProvider != null) { + ret = serviceAccountUsersProvider.isServiceAccountUser(serviceName, userName); + } } - LOG.debug("isDelegationUserForService(serviceName={}, userName={}): ret={}", serviceName, userName, ret); + LOG.debug("isDelegationUserForService(serviceName={}): ret={}", serviceName, ret); return ret; } @@ -506,7 +509,7 @@ private void initializeDelegationUsers() { if (!delegationUsers.isEmpty()) { delegationUsersByService.put(serviceName, delegationUsers); - LOG.info("Delegation users for service '{}': {}", serviceName, delegationUsers); + LOG.info("Configured {} delegation users for service '{}'", delegationUsers.size(), serviceName); } } } diff --git a/pdp/src/main/resources/ranger-pdp-default.xml b/pdp/src/main/resources/ranger-pdp-default.xml index 6f3152f4157..f1c7334c3df 100644 --- a/pdp/src/main/resources/ranger-pdp-default.xml +++ b/pdp/src/main/resources/ranger-pdp-default.xml @@ -271,6 +271,18 @@ How often (ms) to poll Ranger Admin for updated policies and roles. + + ranger.serviceaccount.refresher.enabled + true + Enable periodic retrieval of service-account users from Ranger Admin. + + + + ranger.serviceaccount.refresher.pollIntervalMs + 30000 + How often (ms) to refresh service-account users from Ranger Admin. + + ranger.authz.default.policy.cache.dir /var/ranger/cache/pdp diff --git a/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java b/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java index 1221af3bbdc..da894aefd4f 100644 --- a/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java +++ b/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java @@ -25,6 +25,7 @@ import org.apache.ranger.authz.model.RangerAccessContext; import org.apache.ranger.authz.model.RangerAuthzRequest; import org.apache.ranger.authz.model.RangerAuthzResult; +import org.apache.ranger.authz.model.RangerFilterResourcesRequest; import org.apache.ranger.authz.model.RangerMultiAuthzRequest; import org.apache.ranger.authz.model.RangerMultiAuthzResult; import org.apache.ranger.authz.model.RangerResourcePermissions; @@ -33,6 +34,7 @@ import org.apache.ranger.pdp.RangerPdpStats; import org.apache.ranger.pdp.config.RangerPdpConfig; import org.apache.ranger.pdp.config.RangerPdpConstants; +import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.junit.jupiter.api.Test; import javax.servlet.ServletContext; @@ -41,9 +43,12 @@ import java.lang.reflect.Field; import java.lang.reflect.Proxy; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.Set; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_PDP_SERVICE_PREFIX; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SUFFIX_DELEGATION_USERS; @@ -202,17 +207,54 @@ public void testGetResourcePermissionsReturnsBadRequestOnAuthzException() throws assertEquals(1L, stats.getTotalAuthzBadRequest()); } + @Test + public void testServiceAccountUserCanCallAllPdpApis() throws Exception { + RangerPdpStats stats = new RangerPdpStats(); + TestAuthorizer authorizer = new TestAuthorizer( + new RangerAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW), + new RangerMultiAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW), + new RangerResourcePermissions(), null); + RangerPdpREST rest = createRest(authorizer, stats, serviceAccountProvider(CALLER_DELEGATION_NOT_ALLOWED)); + + assertEquals(Response.Status.OK.getStatusCode(), rest.authorize( + new RangerAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1), + httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + assertEquals(Response.Status.OK.getStatusCode(), rest.authorizeMulti( + new RangerMultiAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1), + httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + assertEquals(Response.Status.OK.getStatusCode(), rest.getResourcePermissions( + new RangerResourcePermissionsRequest("req-1", null, CONTEXT_SERVICE_SVC1), + httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + assertEquals(Response.Status.OK.getStatusCode(), rest.filterResources( + new RangerFilterResourcesRequest("user1", Arrays.asList("table:default/t1"), "_any", "hive", "svc1"), + httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + } + private static RangerPdpREST createRest(TestAuthorizer authorizer, RangerPdpStats stats) throws Exception { + return createRest(authorizer, stats, null); + } + + private static RangerPdpREST createRest(TestAuthorizer authorizer, RangerPdpStats stats, ServiceAccountUsersProvider provider) throws Exception { RangerPdpREST rest = new RangerPdpREST(); setField(rest, "authorizer", authorizer); setField(rest, "config", new TestConfig(defaultConfig())); + setField(rest, "serviceAccountUsersProvider", provider); setField(rest, "servletContext", servletContextWithStats(stats)); rest.initialize(); return rest; } + private static ServiceAccountUsersProvider serviceAccountProvider(String user) throws Exception { + ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, new Properties()); + Map> users = Collections.singletonMap("svc1", Collections.singleton(user)); + + setField(provider, "serviceAccountUsers", users); + + return provider; + } + private static HttpServletRequest httpRequest(String user, RangerPdpStats stats) { Map attrs = new HashMap<>(); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 5bd9d21d2f7..116b33092a9 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -1030,7 +1030,8 @@ public Boolean hasAccess(XXDBBase xxDbBase, RangerBaseModelObject baseModel) { } if (baseModel instanceof RangerServiceHeaderInfo) { - return hasAccessToRangerServiceHeaderInfo((RangerServiceHeaderInfo) baseModel, isKeyAdmin, isSysAdmin, isAuditor, isAuditorKeyAdmin, isUser); + return hasServiceAccountInfoAccess() || + hasAccessToRangerServiceHeaderInfo((RangerServiceHeaderInfo) baseModel, isKeyAdmin, isSysAdmin, isAuditor, isAuditorKeyAdmin, isUser); } return false; diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index 09326c6514c..b38e6f98750 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -437,7 +437,7 @@ public List searchServices(@Context HttpServletRequest request) { @GET @Path("/api/service-headers") @Produces("application/json") - @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_SERVICE_HEADERS + "\")") + @PreAuthorize("@rangerBizUtil.hasServiceAccountInfoAccess() || @rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_SERVICE_HEADERS + "\")") public List getServiceHeaders(@Context HttpServletRequest request) { return serviceREST.getServiceHeaders(request); } diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 4f7bc87acf8..2ca3d8f0513 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -212,6 +212,14 @@ roles. In many cases, this kind of downloading behavior does not need to be protected. + + ranger.admin.service.account.info.viewers + ranger-pdp,ranger-audit-ingestor,admin,rangerauditserver + + Comma-separated users allowed to view service-account users and groups in service-header responses. + Additional users can be appended to this list. + + From d8a41516b505b79f57e2b062dd8113eb1f7e6c83 Mon Sep 17 00:00:00 2001 From: akgupta Date: Tue, 17 Nov 2026 21:45:03 +0530 Subject: [PATCH 2/4] RANGER-5869:Functional tests - Ranger PDP and audit ingestor services to dynamically obtain authorized --- .../configs/ranger-audit-ingestor-site.yaml | 1 + .../audit_ingestor/audit_ingestor_utils.py | 136 +++++++++++++++++ ..._audit_ingestor_authorization_scenarios.py | 60 ++++++++ ...st_audit_ingestor_service_account_users.py | 56 +++++++ functional-tests/pdp/pdp_utils.py | 139 ++++++++++++++++++ .../test_pdp_rest_authorization_scenarios.py | 77 ++++++++++ .../test_pdp_rest_service_account_users.py | 116 +++++++++++++++ functional-tests/pytest.ini | 2 + functional-tests/readme.md | 2 + 9 files changed, 589 insertions(+) create mode 100644 functional-tests/audit_ingestor/audit_ingestor_utils.py create mode 100644 functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py create mode 100644 functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py create mode 100644 functional-tests/pdp/pdp_utils.py create mode 100644 functional-tests/pdp/test_pdp_rest_authorization_scenarios.py create mode 100644 functional-tests/pdp/test_pdp_rest_service_account_users.py diff --git a/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml b/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml index 4a5c3d47801..cef28367bd8 100644 --- a/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml +++ b/dev-support/ranger-docker/scripts/audit-ingestor/configs/ranger-audit-ingestor-site.yaml @@ -29,6 +29,7 @@ ranger.audit.ingestor.contextName: / ranger.audit.ingestor.host: ranger-audit-ingestor.rangernw ranger.audit.ingestor.http.port: '7081' ranger.audit.ingestor.https.port: '7182' +ranger.serviceaccount.refresher.policy.rest.url: 'http://ranger:6080' ranger.audit.ingestor.https.attrib.ssl.enabled: 'false' # HTTPS/SSL Configuration diff --git a/functional-tests/audit_ingestor/audit_ingestor_utils.py b/functional-tests/audit_ingestor/audit_ingestor_utils.py new file mode 100644 index 00000000000..74ae2ac9ac9 --- /dev/null +++ b/functional-tests/audit_ingestor/audit_ingestor_utils.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest +import requests + +from ranger_test_utils.utils import ( + DEFAULT_JSON_HEADERS, + SERVICE_URL, + assert_http_ok, + create_ranger_admin_session, + unique_suffix, +) + +AUDIT_INGESTOR_BASE_URL = os.environ.get( + "RANGER_AUDIT_INGESTOR_BASE_URL", "http://localhost:7081/api/audit" +).rstrip("/") +SERVICE_NAME = "dev_hive" +SERVICE_ACCOUNT_USERS = "service.account.users" +REFRESH_TIMEOUT_SEC = int( + os.environ.get("AUDIT_SERVICE_ACCOUNT_REFRESH_TIMEOUT_SEC", "90") +) + + +class AuditIngestorServiceAccountTestContext: + def __init__(self): + self.admin = create_ranger_admin_session() + self.audit_ingestor = requests.Session() + self.audit_ingestor.headers.update(DEFAULT_JSON_HEADERS) + self.service = None + self.original_value = None + self.user = "audit-functional-" + unique_suffix(8) + self.unauthorized_user = "audit-unauthorized-" + unique_suffix(8) + + def setup(self): + service_url = SERVICE_URL + "name/" + SERVICE_NAME + response = self.admin.get(service_url) + assert_http_ok(response, 200, "Get service " + SERVICE_NAME, service_url) + + self.service = response.json() + configs = self.service.setdefault("configs", {}) + self.original_value = configs.get(SERVICE_ACCOUNT_USERS) + original_users = [ + user.strip() + for user in (self.original_value or "").split(",") + if user.strip() + ] + configs[SERVICE_ACCOUNT_USERS] = ",".join(original_users + [self.user]) + + update_url = SERVICE_URL + str(self.service["id"]) + updated = self.admin.put(update_url, json=self.service) + assert_http_ok( + updated, 200, "Add dynamic service-account user", update_url + ) + self._wait_for_status(200) + + return self + + def cleanup(self): + if self.service is not None: + configs = self.service["configs"] + if self.original_value is None: + configs.pop(SERVICE_ACCOUNT_USERS, None) + else: + configs[SERVICE_ACCOUNT_USERS] = self.original_value + + update_url = SERVICE_URL + str(self.service["id"]) + restored = self.admin.put(update_url, json=self.service) + assert_http_ok( + restored, 200, "Restore service-account users", update_url + ) + self._wait_for_status(403) + + self.audit_ingestor.close() + self.admin.close() + + def post_access( + self, user=None, service_name=SERVICE_NAME, service_type="hive" + ): + self.audit_ingestor.cookies.clear() + return self.audit_ingestor.post( + AUDIT_INGESTOR_BASE_URL + "/access", + params={ + "serviceName": service_name, + "appId": "service-account-functional-test", + }, + headers={"X-Forwarded-User": user or self.user}, + json=[ + { + "repoName": service_name, + "serviceType": service_type, + "user": "audit-functional-target", + "accessType": "select", + "resourcePath": "default/sales", + "accessResult": 1, + "action": "select", + } + ], + timeout=30, + ) + + def _wait_for_status(self, expected_status): + deadline = time.monotonic() + REFRESH_TIMEOUT_SEC + response = None + + while time.monotonic() < deadline: + response = self.post_access() + if response.status_code == expected_status: + return + time.sleep(2) + + pytest.fail( + "Audit Ingestor did not return HTTP " + + str(expected_status) + + " for the dynamic service-account user; last response was " + + ( + str(response.status_code) + ": " + response.text + if response is not None + else "none" + ) + ) diff --git a/functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py b/functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py new file mode 100644 index 00000000000..f5f7414981c --- /dev/null +++ b/functional-tests/audit_ingestor/test_audit_ingestor_authorization_scenarios.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from audit_ingestor.audit_ingestor_utils import ( + AuditIngestorServiceAccountTestContext, +) +from ranger_test_utils.utils import assert_http_ok + +pytestmark = pytest.mark.audit_ingestor + + +@pytest.fixture(scope="module") +def audit_ingestor_context(): + context = AuditIngestorServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def test_unauthorized_user_cannot_post_audits(audit_ingestor_context): + response = audit_ingestor_context.post_access( + user=audit_ingestor_context.unauthorized_user + ) + print("\n[Audit Ingestor] Unauthorized /access returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Unauthorized POST /access") + + +def test_service_account_user_cannot_post_for_another_service( + audit_ingestor_context, +): + response = audit_ingestor_context.post_access( + service_name="dev_hdfs", service_type="hdfs" + ) + print("\n[Audit Ingestor] Cross-service /access returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Cross-service POST /access") + + +def test_configured_user_can_post_audits(audit_ingestor_context): + response = audit_ingestor_context.post_access(user="hive") + print("\n[Audit Ingestor] Configured-user /access returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 200, "Configured-user POST /access") diff --git a/functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py b/functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py new file mode 100644 index 00000000000..f1634b0c63c --- /dev/null +++ b/functional-tests/audit_ingestor/test_audit_ingestor_service_account_users.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from audit_ingestor.audit_ingestor_utils import ( + AuditIngestorServiceAccountTestContext, +) +from ranger_test_utils.utils import ( + assert_http_ok, + get_test_logger, + log_http_response, +) + +pytestmark = pytest.mark.audit_ingestor +logger = get_test_logger(__name__) + + +@pytest.fixture(scope="module") +def audit_ingestor_context(): + context = AuditIngestorServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def test_dynamic_service_account_user_can_post_audits(audit_ingestor_context): + print( + "\n[Audit Ingestor] Testing POST /access " + "with a dynamic service-account user" + ) + response = audit_ingestor_context.post_access() + print( + "[Audit Ingestor] POST /access returned HTTP " + + str(response.status_code) + ) + log_http_response(logger, "POST /access", response) + assert_http_ok(response, 200, "POST /access") + + result = response.json() + assert result["serviceName"] == "dev_hive" + assert result["total"] == 1 diff --git a/functional-tests/pdp/pdp_utils.py b/functional-tests/pdp/pdp_utils.py new file mode 100644 index 00000000000..1ccd6083fac --- /dev/null +++ b/functional-tests/pdp/pdp_utils.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest +import requests + +from ranger_test_utils.utils import ( + DEFAULT_JSON_HEADERS, + SERVICE_URL, + assert_http_ok, + create_ranger_admin_session, + unique_suffix, +) + +PDP_BASE_URL = os.environ.get( + "RANGER_PDP_BASE_URL", "http://localhost:6500/authz/v1" +).rstrip("/") +SERVICE_NAME = "dev_hive" +SERVICE_ACCOUNT_USERS = "service.account.users" +REFRESH_TIMEOUT_SEC = int( + os.environ.get("PDP_SERVICE_ACCOUNT_REFRESH_TIMEOUT_SEC", "90") +) + + +class PdpServiceAccountTestContext: + def __init__(self): + self.admin = create_ranger_admin_session() + self.pdp = requests.Session() + self.pdp.headers.update(DEFAULT_JSON_HEADERS) + self.service = None + self.original_value = None + self.user = "pdp-functional-" + unique_suffix(8) + self.unauthorized_user = "pdp-unauthorized-" + unique_suffix(8) + + def setup(self): + service_url = SERVICE_URL + "name/" + SERVICE_NAME + response = self.admin.get(service_url) + assert_http_ok(response, 200, "Get service " + SERVICE_NAME, service_url) + + self.service = response.json() + configs = self.service.setdefault("configs", {}) + self.original_value = configs.get(SERVICE_ACCOUNT_USERS) + original_users = [ + user.strip() + for user in (self.original_value or "").split(",") + if user.strip() + ] + configs[SERVICE_ACCOUNT_USERS] = ",".join(original_users + [self.user]) + + update_url = SERVICE_URL + str(self.service["id"]) + updated = self.admin.put(update_url, json=self.service) + assert_http_ok( + updated, 200, "Add dynamic service-account user", update_url + ) + self._wait_for_status(200) + + return self + + def cleanup(self): + if self.service is not None: + configs = self.service["configs"] + if self.original_value is None: + configs.pop(SERVICE_ACCOUNT_USERS, None) + else: + configs[SERVICE_ACCOUNT_USERS] = self.original_value + + update_url = SERVICE_URL + str(self.service["id"]) + restored = self.admin.put(update_url, json=self.service) + assert_http_ok( + restored, 200, "Restore service-account users", update_url + ) + self._wait_for_status(403) + + self.pdp.close() + self.admin.close() + + def post(self, endpoint, payload, user=None): + return self.pdp.post( + PDP_BASE_URL + endpoint, + headers={"X-Forwarded-User": user or self.user}, + json=payload, + timeout=30, + ) + + @staticmethod + def context(service_name=SERVICE_NAME, service_type="hive"): + return {"serviceType": service_type, "serviceName": service_name} + + @staticmethod + def access(resource): + return { + "resource": {"name": resource}, + "action": "QUERY", + "permissions": ["select"], + } + + def _wait_for_status(self, expected_status): + deadline = time.monotonic() + REFRESH_TIMEOUT_SEC + response = None + + while time.monotonic() < deadline: + response = self.post( + "/authorize", + { + "requestId": "service-account-refresh-check", + "user": {"name": "hive"}, + "access": self.access("table:default/sales"), + "context": self.context(), + }, + ) + if response.status_code == expected_status: + return + time.sleep(2) + + pytest.fail( + "PDP did not return HTTP " + + str(expected_status) + + " for the dynamic service-account user; last response was " + + ( + str(response.status_code) + ": " + response.text + if response is not None + else "none" + ) + ) diff --git a/functional-tests/pdp/test_pdp_rest_authorization_scenarios.py b/functional-tests/pdp/test_pdp_rest_authorization_scenarios.py new file mode 100644 index 00000000000..8d8af6973c4 --- /dev/null +++ b/functional-tests/pdp/test_pdp_rest_authorization_scenarios.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pdp.pdp_utils import PdpServiceAccountTestContext +from ranger_test_utils.utils import assert_http_ok + +pytestmark = pytest.mark.pdp + + +@pytest.fixture(scope="module") +def pdp_context(): + context = PdpServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def test_unauthorized_user_cannot_authorize(pdp_context): + response = pdp_context.post( + "/authorize", + { + "requestId": "pdp-functional-unauthorized", + "user": {"name": "hive"}, + "access": pdp_context.access("table:default/sales"), + "context": pdp_context.context(), + }, + pdp_context.unauthorized_user, + ) + print("\n[PDP] Unauthorized /authorize returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Unauthorized POST /authorize") + + +def test_service_account_user_cannot_authorize_another_service(pdp_context): + response = pdp_context.post( + "/authorizeMulti", + { + "requestId": "pdp-functional-wrong-service", + "user": {"name": "hdfs"}, + "accesses": [pdp_context.access("/tmp/test")], + "context": pdp_context.context("dev_hdfs", "hdfs"), + }, + ) + print("\n[PDP] Cross-service /authorizeMulti returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 403, "Cross-service POST /authorizeMulti") + + +def test_configured_user_can_get_permissions(pdp_context): + response = pdp_context.post( + "/permissions", + { + "requestId": "pdp-functional-configured-user", + "resource": {"name": "table:default/sales"}, + "context": pdp_context.context(), + }, + "ranger", + ) + print("\n[PDP] Configured-user /permissions returned HTTP " + + str(response.status_code)) + assert_http_ok(response, 200, "Configured-user POST /permissions") diff --git a/functional-tests/pdp/test_pdp_rest_service_account_users.py b/functional-tests/pdp/test_pdp_rest_service_account_users.py new file mode 100644 index 00000000000..3f029e7d7e2 --- /dev/null +++ b/functional-tests/pdp/test_pdp_rest_service_account_users.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pdp.pdp_utils import PdpServiceAccountTestContext +from ranger_test_utils.utils import ( + assert_http_ok, + get_test_logger, + log_http_response, +) + +pytestmark = pytest.mark.pdp +logger = get_test_logger(__name__) + + +@pytest.fixture(scope="module") +def pdp_context(): + context = PdpServiceAccountTestContext() + + try: + yield context.setup() + finally: + context.cleanup() + + +def post(pdp_context, endpoint, payload): + print("\n[PDP] Testing POST " + endpoint + " with a dynamic service-account user") + response = pdp_context.post(endpoint, payload) + print("[PDP] POST " + endpoint + " returned HTTP " + str(response.status_code)) + log_http_response(logger, "POST " + endpoint, response) + assert_http_ok(response, 200, "POST " + endpoint) + + return response.json() + + +def test_dynamic_service_account_user_can_authorize(pdp_context): + result = post( + pdp_context, + "/authorize", + { + "requestId": "pdp-functional-authorize", + "user": {"name": "hive"}, + "access": pdp_context.access("table:default/sales"), + "context": pdp_context.context(), + }, + ) + + assert result["requestId"] == "pdp-functional-authorize" + assert result["decision"] in ("ALLOW", "DENY") + + +def test_dynamic_service_account_user_can_authorize_multi(pdp_context): + result = post( + pdp_context, + "/authorizeMulti", + { + "requestId": "pdp-functional-authorize-multi", + "user": {"name": "hive"}, + "accesses": [ + pdp_context.access("table:default/sales"), + pdp_context.access("table:default/customers"), + ], + "context": pdp_context.context(), + }, + ) + + assert result["requestId"] == "pdp-functional-authorize-multi" + assert len(result["accesses"]) == 2 + + +def test_dynamic_service_account_user_can_get_permissions(pdp_context): + result = post( + pdp_context, + "/permissions", + { + "requestId": "pdp-functional-permissions", + "resource": {"name": "table:default/sales"}, + "context": pdp_context.context(), + }, + ) + + assert result["resource"]["name"] == "table:default/sales" + + +def test_dynamic_service_account_user_can_filter_resources(pdp_context): + result = post( + pdp_context, + "/filterResources", + { + "requestId": "pdp-functional-filter-resources", + "user": {"name": "hive"}, + "resources": [ + {"name": "table:default/sales"}, + {"name": "table:default/customers"}, + ], + "action": "select", + "permissions": ["select"], + "context": pdp_context.context(), + }, + ) + + assert result["requestId"] == "pdp-functional-filter-resources" + assert "resources" in result diff --git a/functional-tests/pytest.ini b/functional-tests/pytest.ini index d17dbeb90db..feb8b3dd780 100644 --- a/functional-tests/pytest.ini +++ b/functional-tests/pytest.ini @@ -32,6 +32,8 @@ markers = hbase: HBase plugin enforcement (Docker HBase shell) hdfs_policy: HDFS Ranger plugin path policy enforcement (Docker dfs) kafka: Kafka plugin enforcement (Docker Kafka console producer) + pdp: PDP live REST API tests + audit_ingestor: Audit Ingestor live REST API tests pythonpath = . # No console logging during pytest; logger.info -> logs/.log (see conftest.py). log_cli = false diff --git a/functional-tests/readme.md b/functional-tests/readme.md index b11915cce4b..b50a0e7f59e 100644 --- a/functional-tests/readme.md +++ b/functional-tests/readme.md @@ -36,6 +36,8 @@ A Docker-based test execution framework that validates Apache Ranger service int | **hbase** | HBase Ranger plugin enforcement via HBase shell (requires `ranger-hadoop` + `ranger-hbase`) | | **jvm_locale_test** | Ranger Admin JVM locale tests. Run with `run-jvm-locale-tests.sh`. | | **kafka** | Kafka Ranger plugin enforcement via console producer (requires `ranger-kafka`) | +| **pdp** | Live PDP REST authorization using dynamically refreshed service-account users | +| **audit_ingestor** | Live Audit Ingestor REST authorization using dynamically refreshed service-account users | ### Directory Structure From 9daafde456287f237f4d56a30f030509143241ff Mon Sep 17 00:00:00 2001 From: akgupta Date: Wed, 18 Nov 2026 10:47:09 +0530 Subject: [PATCH 3/4] RANGER-5869:Ranger PDP and audit ingestor services to dynamically obtain authorized callers bootstrap refresh changes Co-authored-by: Cursor --- .../util/ServiceAccountUsersProvider.java | 99 ---------- .../util/ServiceAccountUsersRefresher.java | 146 --------------- .../util/TestServiceAccountUsersProvider.java | 86 --------- .../audit/server/AuditServerConstants.java | 2 +- .../apache/ranger/audit/rest/AuditREST.java | 122 ++++++++++-- .../AuditServiceAccountRefreshBootstrap.java | 42 +++++ .../AuditServiceAccountUsersConfig.java | 46 ----- .../ranger/audit/server/EmbeddedServer.java | 3 + .../conf/ranger-audit-ingestor-site.xml | 54 ++++-- .../ranger/audit/rest/AuditRESTTest.java | 120 ++++++------ ...ditServiceAccountRefreshBootstrapTest.java | 36 ++++ distro/src/main/assembly/pdp.xml | 13 ++ pdp/conf.dist/ranger-pdp-site.xml | 22 +-- .../apache/ranger/pdp/RangerPdpServer.java | 27 +-- .../ranger/pdp/config/RangerPdpConfig.java | 23 --- .../ranger/pdp/config/RangerPdpConstants.java | 4 +- .../PdpServiceAccountRefreshBootstrap.java | 49 +++++ .../ranger/pdp/rest/RangerPdpApplication.java | 20 +- .../apache/ranger/pdp/rest/RangerPdpREST.java | 173 +++++++++++++++--- pdp/src/main/resources/ranger-pdp-default.xml | 22 +-- ...PdpServiceAccountRefreshBootstrapTest.java | 53 ++++++ .../ranger/pdp/rest/RangerPdpRESTTest.java | 94 ++++++++-- .../conf.dist/ranger-admin-default-site.xml | 4 +- 23 files changed, 654 insertions(+), 606 deletions(-) delete mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java delete mode 100644 agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java delete mode 100644 agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java create mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java delete mode 100644 audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java create mode 100644 audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java create mode 100644 pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java create mode 100644 pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java deleted file mode 100644 index e0531a1290e..00000000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersProvider.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.plugin.util; - -import org.apache.hadoop.conf.Configuration; -import org.apache.ranger.admin.client.RangerAdminRESTClient; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import static java.util.Objects.requireNonNull; - -public final class ServiceAccountUsersProvider { - private static final String CONFIG_PREFIX = "ranger.serviceaccount.refresher"; - private static final String POLLING_INTERVAL_PROPERTY = CONFIG_PREFIX + ".pollIntervalMs"; - - private final ServiceAccountUsersRefresher refresher; - - private volatile Map> serviceAccountUsers = Collections.emptyMap(); - - public ServiceAccountUsersProvider(boolean serviceAccountRefreshEnabled, Configuration config) { - refresher = serviceAccountRefreshEnabled ? createRefresher(requireNonNull(config, "config")) : null; - } - - public ServiceAccountUsersProvider(boolean serviceAccountRefreshEnabled, Properties properties) { - this(serviceAccountRefreshEnabled, serviceAccountRefreshEnabled ? toConfiguration(properties) : null); - } - - public void start() { - if (refresher != null) { - refresher.startRefresher(); - } - } - - public void stop() { - if (refresher != null) { - refresher.stopRefresher(); - } - } - - public boolean isServiceAccountUser(String serviceName, String userName) { - Set users = serviceAccountUsers.get(serviceName); - - return users != null && users.contains(userName); - } - - private ServiceAccountUsersRefresher createRefresher(Configuration config) { - RangerAdminRESTClient rangerAdmin = new RangerAdminRESTClient(); - - rangerAdmin.init("service-account-users", "service-account-users-provider", CONFIG_PREFIX, config); - - return new ServiceAccountUsersRefresher(rangerAdmin::getServiceAccountUsers, this::setServiceAccountUsers, - config.getLong(POLLING_INTERVAL_PROPERTY, -1L)); - } - - private static Configuration toConfiguration(Properties properties) { - Configuration ret = new Configuration(false); - - Properties source = requireNonNull(properties, "properties"); - - for (String name : source.stringPropertyNames()) { - ret.set(name, source.getProperty(name)); - } - - return ret; - } - - private void setServiceAccountUsers(Map> serviceAccountUsers) { - Map> snapshot = new HashMap<>(); - - serviceAccountUsers.forEach((serviceName, users) -> { - if (serviceName != null && users != null && !users.isEmpty()) { - snapshot.put(serviceName, Collections.unmodifiableSet(new HashSet<>(users))); - } - }); - - this.serviceAccountUsers = Collections.unmodifiableMap(snapshot); - } -} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java deleted file mode 100644 index 889db1a96f7..00000000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceAccountUsersRefresher.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.plugin.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.Set; -import java.util.Timer; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.function.Consumer; - -import static java.util.Objects.requireNonNull; - -final class ServiceAccountUsersRefresher extends Thread { - private static final Logger LOG = LoggerFactory.getLogger(ServiceAccountUsersRefresher.class); - - private final Callable>> serviceAccountUsersLoader; - private final Consumer>> receiver; - private final long pollingIntervalMs; - private final BlockingQueue downloadQueue = new LinkedBlockingQueue<>(); - private Timer downloadTimer; - - ServiceAccountUsersRefresher(Callable>> serviceAccountUsersLoader, Consumer>> receiver, long pollingIntervalMs) { - if (pollingIntervalMs <= 0) { - throw new IllegalArgumentException("pollingIntervalMs must be greater than zero"); - } - - this.serviceAccountUsersLoader = requireNonNull(serviceAccountUsersLoader, "serviceAccountUsersLoader"); - this.receiver = requireNonNull(receiver, "receiver"); - this.pollingIntervalMs = pollingIntervalMs; - - setName("ServiceAccountUsersRefresher-" + getId()); - setDaemon(true); - } - - void startRefresher() { - refresh(); - - try { - start(); - - downloadTimer = new Timer("serviceAccountUsersDownloadTimer", true); - downloadTimer.schedule(new DownloaderTask(downloadQueue), pollingIntervalMs, pollingIntervalMs); - - LOG.info("Started service-account user refresher with polling interval {} ms", pollingIntervalMs); - } catch (RuntimeException excp) { - LOG.error("Failed to start service-account user refresher", excp); - - stopRefresher(); - - throw excp; - } - } - - void stopRefresher() { - boolean wasStarted = downloadTimer != null || isAlive(); - Timer downloadTimer = this.downloadTimer; - - this.downloadTimer = null; - - if (downloadTimer != null) { - downloadTimer.cancel(); - } - - if (isAlive()) { - interrupt(); - - boolean interrupted = false; - boolean joined = false; - - while (!joined) { - try { - join(); - - joined = true; - } catch (InterruptedException excp) { - LOG.warn("Interrupted while waiting for service-account user refresher to stop", excp); - - interrupted = true; - } - } - - if (interrupted) { - Thread.currentThread().interrupt(); - } - } - - if (wasStarted) { - LOG.info("Stopped service-account user refresher"); - } - } - - @Override - public void run() { - while (true) { - DownloadTrigger trigger = null; - - try { - trigger = downloadQueue.take(); - - refresh(); - } catch (InterruptedException excp) { - LOG.info("Service-account user refresher interrupted; exiting"); - - break; - } finally { - if (trigger != null) { - trigger.signalCompletion(); - } - } - } - } - - void refresh() { - try { - Map> serviceAccountUsers = requireNonNull(serviceAccountUsersLoader.call(), - "Ranger Admin returned null service-account users"); - - receiver.accept(serviceAccountUsers); - - LOG.debug("Refreshed service-account users for {} services", serviceAccountUsers.size()); - } catch (Exception excp) { - LOG.warn("Failed to refresh service-account users; retaining the last successful snapshot", excp); - } - } -} diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java b/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java deleted file mode 100644 index 441f22ceb39..00000000000 --- a/agents-common/src/test/java/org/apache/ranger/plugin/util/TestServiceAccountUsersProvider.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.plugin.util; - -import org.apache.hadoop.conf.Configuration; -import org.junit.jupiter.api.Test; - -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestServiceAccountUsersProvider { - @Test - public void testProviderLoadsDefensiveSnapshot() throws Exception { - Set users = new HashSet<>(Collections.singleton("alice")); - Map> source = new HashMap<>(); - ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, (Configuration) null); - Method setUsers = ServiceAccountUsersProvider.class.getDeclaredMethod("setServiceAccountUsers", Map.class); - - source.put("svc", users); - - setUsers.setAccessible(true); - setUsers.invoke(provider, source); - users.add("bob"); - - assertTrue(provider.isServiceAccountUser("svc", "alice")); - assertFalse(provider.isServiceAccountUser("svc", "bob")); - - setUsers.invoke(provider, Collections.emptyMap()); - assertFalse(provider.isServiceAccountUser("svc", "alice")); - } - - @Test - public void testRefresherReplacesSnapshotAndRetainsItOnFailure() { - AtomicInteger calls = new AtomicInteger(); - AtomicReference>> received = new AtomicReference<>(); - ServiceAccountUsersRefresher refresher = new ServiceAccountUsersRefresher(() -> { - if (calls.getAndIncrement() == 0) { - return Collections.singletonMap("svc", Collections.singleton("alice")); - } - - throw new Exception("expected"); - }, received::set, 60_000L); - - refresher.refresh(); - Map> snapshot = received.get(); - refresher.refresh(); - - assertSame(snapshot, received.get()); - } - - @Test - public void testDisabledProviderIsNoOp() { - ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, (Configuration) null); - - assertDoesNotThrow(provider::start); - assertFalse(provider.isServiceAccountUser("svc", "alice")); - assertDoesNotThrow(provider::stop); - } -} diff --git a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java index d79202e4cac..fc9b290031a 100644 --- a/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java +++ b/audit-server/audit-common/src/main/java/org/apache/ranger/audit/server/AuditServerConstants.java @@ -27,7 +27,7 @@ private AuditServerConstants() {} public static final String PROP_PREFIX_AUDIT_SERVER_SERVICE = PROP_PREFIX_AUDIT_SERVER + "service."; public static final String PROP_AUTH_TO_LOCAL = PROP_PREFIX_AUDIT_SERVER + "auth.to.local"; public static final String PROP_SUFFIX_ALLOWED_USERS = ".allowed.users"; - public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = "ranger.serviceaccount.refresher.enabled"; + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_PREFIX_AUDIT_SERVER + "serviceaccount.refresher.enabled"; public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "service.kerberos.principal"; public static final String PROP_AUDIT_SERVICE_KEYTAB = "service.kerberos.keytab"; diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java index af334c74970..d4fd97d98f5 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java @@ -20,13 +20,14 @@ package org.apache.ranger.audit.rest; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.audit.model.AuthzAuditEvent; import org.apache.ranger.audit.producer.AuditDestinationMgr; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.audit.server.AuditServerConfig; import org.apache.ranger.audit.server.AuditServerConstants; -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -44,10 +45,14 @@ import javax.ws.rs.core.Response; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.stream.Collectors; import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE; @@ -59,18 +64,111 @@ public class AuditREST { private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class); - private static final Map> allowedServiceUsers; + private static final String RANGER_ADMIN_CONFIG_PREFIX = "ranger.audit.ingestor"; + private static final String PROP_POLICY_REST_URL = RANGER_ADMIN_CONFIG_PREFIX + ".policy.rest.url"; + private static final String PROP_POLICY_POLL_INTERVAL_MS = RANGER_ADMIN_CONFIG_PREFIX + ".policy.pollIntervalMs"; + private static final long DEFAULT_POLL_INTERVAL_MS = 30 * 1000L; + + private static final Map> configuredAllowedUsers; + + private static volatile Map> allowedServiceUsers; + private static RangerAdminRESTClient rangerAdminClient; + private static Timer serviceAccountUsersRefreshTimer; static { - allowedServiceUsers = initializeAllowedUsers(); + configuredAllowedUsers = initializeAllowedUsers(); + allowedServiceUsers = mergeAllowedUsers(configuredAllowedUsers, Collections.emptyMap()); initializeAuthToLocal(); } - @Autowired - AuditDestinationMgr auditDestinationMgr; + public static synchronized void startServiceAccountUsersRefresh(Configuration configuration) { + if (!configuration.getBoolean(AuditServerConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, false)) { + return; + } + + String rangerAdminUrl = configuration.get(PROP_POLICY_REST_URL); + + if (serviceAccountUsersRefreshTimer == null && StringUtils.isNotBlank(rangerAdminUrl)) { + try { + RangerAdminRESTClient client = new RangerAdminRESTClient(); + + client.init("audit-ingestor", "ranger-audit", RANGER_ADMIN_CONFIG_PREFIX, configuration); + + rangerAdminClient = client; + + long pollIntervalMs = configuration.getLong(PROP_POLICY_POLL_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS); + Timer timer = new Timer("auditServiceAccountUsersRefresh", true); + + timer.schedule(new TimerTask() { + @Override + public void run() { + refreshAllowedUsers(rangerAdminClient); + } + }, pollIntervalMs, pollIntervalMs); + + serviceAccountUsersRefreshTimer = timer; + + LOG.info("Scheduled service account user refresh every {} ms from {}", pollIntervalMs, rangerAdminUrl); + refreshAllowedUsers(client); + } catch (Exception e) { + LOG.warn("Failed to start service account user refresh", e); + } + } + } + + public static synchronized void stopServiceAccountUsersRefresh() { + Timer timer = serviceAccountUsersRefreshTimer; + + serviceAccountUsersRefreshTimer = null; + + if (timer != null) { + timer.cancel(); + } + } + + static void refreshAllowedUsers(RangerAdminRESTClient client) { + if (client != null) { + try { + Map> serviceAccountUsers = client.getServiceAccountUsers(); + allowedServiceUsers = mergeAllowedUsers(configuredAllowedUsers, serviceAccountUsers); + if (!serviceAccountUsers.isEmpty()) { + LOG.info("Merged service.account.users for {} service(s) from Ranger Admin", serviceAccountUsers.size()); + } + } catch (Exception e) { + LOG.warn("Failed to refresh service account users from Ranger Admin", e); + } + } + } + + static Map> mergeAllowedUsers(Map> configuredAllowedUsers, Map> serviceAccountUsersByServiceName) { + Map> ret = new HashMap<>(); + + for (Map.Entry> entry : configuredAllowedUsers.entrySet()) { + ret.put(entry.getKey(), new HashSet<>(entry.getValue())); + } + + for (Map.Entry> entry : serviceAccountUsersByServiceName.entrySet()) { + String serviceName = entry.getKey(); + Set serviceAccountUsers = entry.getValue(); + + if (StringUtils.isNotBlank(serviceName) && serviceAccountUsers != null && !serviceAccountUsers.isEmpty()) { + Set allowedUsers = ret.get(serviceName); + + if (allowedUsers == null) { + allowedUsers = new HashSet<>(); + + ret.put(serviceName, allowedUsers); + } + + allowedUsers.addAll(serviceAccountUsers); + } + } + + return ret; + } @Autowired - ServiceAccountUsersProvider serviceAccountUsersProvider; + AuditDestinationMgr auditDestinationMgr; /** * Health check endpoint @@ -289,6 +387,10 @@ private String applyAuthToLocal(String principal) { return principal; } + if (principal.startsWith("spiffe://")) { + return principal; + } + // Check if this looks like a Kerberos principal (has @ or /) if (!principal.contains("@") && !principal.contains("/")) { LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal); @@ -337,10 +439,6 @@ private boolean isAllowedServiceUser(String serviceName, String userName) { Set allowedUsers = allowedServiceUsers.get(serviceName); ret = allowedUsers != null && allowedUsers.contains(userName); - - if (!ret && serviceAccountUsersProvider != null) { - ret = serviceAccountUsersProvider.isServiceAccountUser(serviceName, userName); - } } else { ret = false; } @@ -350,10 +448,6 @@ private boolean isAllowedServiceUser(String serviceName, String userName) { return ret; } - /** - * Initialize the set of allowed service users from configuration - * @return Map of allowed usernames per service - */ private static Map> initializeAllowedUsers() { Map> ret = new HashMap<>(); AuditServerConfig config = AuditServerConfig.getInstance(); diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java new file mode 100644 index 00000000000..2c36638f0be --- /dev/null +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrap.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.audit.rest; + +import org.apache.ranger.audit.server.AuditServerConfig; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +/** + * Starts service-account refresh from the Spring webapp class loader so {@link AuditREST} + * static allow maps stay in sync with the JAX-RS beans that authorize ingest traffic. + */ +@Component +public class AuditServiceAccountRefreshBootstrap { + @PostConstruct + public void start() { + AuditREST.startServiceAccountUsersRefresh(AuditServerConfig.getInstance()); + } + + @PreDestroy + public void stop() { + AuditREST.stopServiceAccountUsersRefresh(); + } +} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java deleted file mode 100644 index 6a12ec9d36a..00000000000 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/AuditServiceAccountUsersConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ranger.audit.server; - -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class AuditServiceAccountUsersConfig { - private static final Logger LOG = LoggerFactory.getLogger(AuditServiceAccountUsersConfig.class); - - @Bean(initMethod = "start", destroyMethod = "stop") - public ServiceAccountUsersProvider serviceAccountUsersProvider() { - AuditServerConfig config = AuditServerConfig.getInstance(); - boolean enabled = config.getBoolean(AuditServerConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, true); - - try { - LOG.info("Service-account user refresher is configured as {}", enabled ? "enabled" : "disabled"); - - return new ServiceAccountUsersProvider(enabled, config); - } catch (RuntimeException excp) { - LOG.error("Failed to initialize service-account user refresher; continuing with configured allowed users", excp); - - return new ServiceAccountUsersProvider(false, config); - } - } -} diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java index cde2b8267aa..c3018e53ac6 100644 --- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java +++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/server/EmbeddedServer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.apache.hadoop.security.alias.LocalBouncyCastleFipsKeyStoreProvider; import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.audit.rest.AuditREST; import org.apache.ranger.audit.utils.AuditServerLogFormatter; import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider; import org.apache.tomcat.util.net.SSLHostConfig; @@ -473,6 +474,8 @@ private void addShutdownHook() { private void gracefulShutdown() { LOG.info("==> EmbeddedServer.gracefulShutdown()"); + AuditREST.stopServiceAccountUsersRefresh(); + // Instead of trying to manually shutdown Spring context, let's rely on Tomcat's // normal webapp lifecycle which will trigger ContextLoaderListener.contextDestroyed() // and call @PreDestroy methods automatically diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml index 3b8a681d424..b72aa8a0dd1 100644 --- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml +++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml @@ -50,24 +50,6 @@ 7182 - - ranger.serviceaccount.refresher.enabled - true - Enable periodic retrieval of service-account users from Ranger Admin. - - - - ranger.serviceaccount.refresher.policy.rest.url - http://localhost:6080 - Ranger Admin base URL used to retrieve service-account users. - - - - ranger.serviceaccount.refresher.pollIntervalMs - 30000 - How often (ms) to refresh service-account users from Ranger Admin. - - ranger.audit.ingestor.https.attrib.ssl.enabled false @@ -223,6 +205,42 @@ Comma-separated list of HTTP header name(s) carrying a SPIFFE ID (e.g. spiffe://cluster/ns/<ns>/sa/<service-account>). A single header name is also valid. The full SPIFFE ID is used as the authenticated principal. When multiple headers are configured, the first one carrying a well-formed SPIFFE ID is used. Consulted only when the username header is absent. + + ranger.audit.ingestor.serviceaccount.refresher.enabled + false + When true and ranger.audit.ingestor.policy.rest.url is set, periodically merge service.account.users from Ranger Admin into allowed audit callers. Override via Cloudera Manager safety valve. + + + + ranger.audit.ingestor.policy.rest.url + + Ranger Admin base URL for service-account refresh. Uses ranger.audit.ingestor.policy.rest.ssl.config.file and .client.username/.password for authentication (safety-valve overrides). Leave empty to use only ranger.audit.ingestor.service.<serviceName>.allowed.users. Poll identity must be in ranger.admin.service.account.info.viewers. + + + + ranger.audit.ingestor.policy.pollIntervalMs + 30000 + How often (ms) to refresh service.account.users when ranger.audit.ingestor.serviceaccount.refresher.enabled is true. + + + + ranger.audit.ingestor.policy.rest.ssl.config.file + + SSL config file for the Ranger Admin connection. Required when ranger.audit.ingestor.policy.rest.url uses https. + + + + ranger.audit.ingestor.policy.rest.client.username + + Username for the Ranger Admin connection (basic auth). + + + + ranger.audit.ingestor.policy.rest.client.password + + Password for the Ranger Admin connection (basic auth). + + ranger.audit.ingestor.service.dev_hdfs.allowed.users hdfs diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java index cdf8467e711..58ea6eb6552 100644 --- a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditRESTTest.java @@ -7,10 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. @@ -18,96 +18,84 @@ package org.apache.ranger.audit.rest; -import org.apache.ranger.audit.model.AuditEventBase; -import org.apache.ranger.audit.model.AuthzAuditEvent; -import org.apache.ranger.audit.producer.AuditDestinationMgr; -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.junit.jupiter.api.Test; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.core.Response; - import java.lang.reflect.Field; -import java.lang.reflect.Proxy; -import java.security.Principal; -import java.util.Collection; +import java.util.Arrays; import java.util.Collections; -import java.util.List; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; -import java.util.Properties; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; class AuditRESTTest { - private static final String SERVICE = "dynamic-service"; - private static final String USER = "dynamic-user"; - @Test - void testServiceAccountUserCanPostAudits() throws Exception { - TestDestination destination = new TestDestination(); - List audits = Collections.singletonList(new AuthzAuditEvent()); - AuditREST rest = createRest(destination, provider(true)); + void testMergeAllowedUsersUnionsServiceAccountUsers() { + Map> configuredAllowedUsers = new HashMap<>(); - Response response = rest.logAccessAudit(SERVICE, "app", audits, request(USER)); + configuredAllowedUsers.put("dev_hive", new HashSet<>(Collections.singleton("hive"))); - assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - assertTrue(destination.called); - } + Map> serviceAccountUsersByServiceName = new HashMap<>(); - @Test - void testUnknownUserCannotPostAudits() throws Exception { - TestDestination destination = new TestDestination(); - List audits = Collections.singletonList(new AuthzAuditEvent()); - AuditREST rest = createRest(destination, provider(false)); + serviceAccountUsersByServiceName.put("dev_hive", Collections.singleton("hive2")); + serviceAccountUsersByServiceName.put("dev_kafka", Collections.singleton("kafka")); - Response response = rest.logAccessAudit(SERVICE, "app", audits, request(USER)); + Map> mergedAllowedUsers = AuditREST.mergeAllowedUsers(configuredAllowedUsers, serviceAccountUsersByServiceName); - assertEquals(Response.Status.FORBIDDEN.getStatusCode(), response.getStatus()); - assertFalse(destination.called); + assertEquals(new HashSet<>(Arrays.asList("hive", "hive2")), mergedAllowedUsers.get("dev_hive")); + assertEquals(Collections.singleton("kafka"), mergedAllowedUsers.get("dev_kafka")); } - private static AuditREST createRest(AuditDestinationMgr destination, ServiceAccountUsersProvider provider) { - AuditREST ret = new AuditREST(); + @Test + void testRefreshKeepsAllowedUsersWhenAdminCallFails() throws Exception { + Map> before = allowedUsers(); - ret.auditDestinationMgr = destination; - ret.serviceAccountUsersProvider = provider; + AuditREST.refreshAllowedUsers(new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() throws Exception { + throw new Exception("admin down"); + } + }); - return ret; + assertSame(before, allowedUsers()); } - private static HttpServletRequest request(String user) { - Principal principal = () -> user; - - return (HttpServletRequest) Proxy.newProxyInstance( - HttpServletRequest.class.getClassLoader(), - new Class[] {HttpServletRequest.class}, - (proxy, method, args) -> "getUserPrincipal".equals(method.getName()) ? principal : null); + @Test + void testRefreshAddsServiceAccountUsers() throws Exception { + Map> before = allowedUsers(); + + try { + AuditREST.refreshAllowedUsers(new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.singletonMap("dev_new", Collections.singleton("new-user")); + } + }); + + assertTrue(allowedUsers().get("dev_new").contains("new-user")); + } finally { + AuditREST.refreshAllowedUsers(new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.emptyMap(); + } + }); + + assertEquals(before.keySet(), allowedUsers().keySet()); + } } - private static ServiceAccountUsersProvider provider(boolean allowed) throws Exception { - ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, new Properties()); - Map> users = allowed - ? Collections.singletonMap(SERVICE, Collections.singleton(USER)) - : Collections.emptyMap(); - Field field = ServiceAccountUsersProvider.class.getDeclaredField("serviceAccountUsers"); + @SuppressWarnings("unchecked") + private static Map> allowedUsers() throws Exception { + Field field = AuditREST.class.getDeclaredField("allowedServiceUsers"); field.setAccessible(true); - field.set(provider, users); - - return provider; - } - private static class TestDestination extends AuditDestinationMgr { - private boolean called; - - @Override - public boolean logBatch(Collection events, String appId) { - called = true; - - return true; - } + return (Map>) field.get(null); } } diff --git a/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java new file mode 100644 index 00000000000..b92f348e1ff --- /dev/null +++ b/audit-server/audit-ingestor/src/test/java/org/apache/ranger/audit/rest/AuditServiceAccountRefreshBootstrapTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.rest; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class AuditServiceAccountRefreshBootstrapTest { + @Test + void lifecycleWithRefresherDisabledDoesNotThrow() { + AuditServiceAccountRefreshBootstrap bootstrap = new AuditServiceAccountRefreshBootstrap(); + + assertDoesNotThrow(() -> { + bootstrap.start(); + bootstrap.stop(); + }); + } +} diff --git a/distro/src/main/assembly/pdp.xml b/distro/src/main/assembly/pdp.xml index 37fad91fa4b..cd9b2e5938b 100644 --- a/distro/src/main/assembly/pdp.xml +++ b/distro/src/main/assembly/pdp.xml @@ -146,6 +146,19 @@ + + + + lib + false + false + + org.apache.ranger:ranger-audit-core:jar:${project.version} + org.apache.ranger:ranger-audit-dest-auditserver:jar:${project.version} + + + + diff --git a/pdp/conf.dist/ranger-pdp-site.xml b/pdp/conf.dist/ranger-pdp-site.xml index f1c7334c3df..88ac3a6ad68 100644 --- a/pdp/conf.dist/ranger-pdp-site.xml +++ b/pdp/conf.dist/ranger-pdp-site.xml @@ -230,12 +230,20 @@ Policy source implementation. Use RangerAdminRESTClient to download policies live from Ranger Admin. + + ranger.pdp.serviceaccount.refresher.enabled + false + When true and ranger.authz.default.policy.rest.url is set, periodically merge service.account.users from Ranger Admin into PDP delegation users. Override via Cloudera Manager safety valve. Poll credentials use ranger.authz.default.policy.rest.* (same as policy download). + + ranger.authz.default.policy.rest.url http://localhost:6080 URL of the Ranger Admin server. Comma-separated for HA. Example: http://ranger-admin-1:6080,http://ranger-admin-2:6080 + Also used for service-account user refresh when ranger.pdp.serviceaccount.refresher.enabled is true (rest client auth via ranger.authz.default.policy.rest.ssl.config.file and .client.username/.password). + The REST client identity must be listed in ranger.admin.service.account.info.viewers on Admin. @@ -268,19 +276,7 @@ ranger.authz.default.policy.pollIntervalMs 30000 - How often (ms) to poll Ranger Admin for updated policies and roles. - - - - ranger.serviceaccount.refresher.enabled - true - Enable periodic retrieval of service-account users from Ranger Admin. - - - - ranger.serviceaccount.refresher.pollIntervalMs - 30000 - How often (ms) to refresh service-account users from Ranger Admin. + How often (ms) to poll Ranger Admin for updated policies and roles and to refresh service-account users when ranger.pdp.serviceaccount.refresher.enabled is true. diff --git a/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java b/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java index 7ee5421eb77..2f7ba67b199 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/RangerPdpServer.java @@ -33,7 +33,6 @@ import org.apache.ranger.pdp.rest.RangerPdpApplication; import org.apache.ranger.pdp.security.RangerPdpAuthNFilter; import org.apache.ranger.pdp.security.RangerPdpRequestContextFilter; -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.apache.tomcat.util.descriptor.web.FilterDef; import org.apache.tomcat.util.descriptor.web.FilterMap; import org.glassfish.jersey.servlet.ServletContainer; @@ -66,7 +65,6 @@ public class RangerPdpServer { private final RangerPdpStats runtimeStats = new RangerPdpStats(); private Tomcat tomcat; private RangerAuthorizer authorizer; - private ServiceAccountUsersProvider serviceAccountUsersProvider; public RangerPdpServer() { this.config = new RangerPdpConfig(); @@ -80,7 +78,6 @@ public void start() throws Exception { LOG.info("Starting Ranger PDP server"); initAuthorizer(); - initServiceAccountUsersProvider(); startTomcat(); } @@ -110,10 +107,6 @@ public void stop() { LOG.warn("Error closing authorizer", e); } } - - if (serviceAccountUsersProvider != null) { - serviceAccountUsersProvider.stop(); - } } private void initAuthorizer() throws RangerAuthzException { @@ -128,24 +121,6 @@ private void initAuthorizer() throws RangerAuthzException { LOG.info("RangerEmbeddedAuthorizer initialised"); } - private void initServiceAccountUsersProvider() { - boolean enabled = config.isServiceAccountRefresherEnabled(); - ServiceAccountUsersProvider provider; - - try { - provider = new ServiceAccountUsersProvider(enabled, config.getServiceAccountRefresherProperties()); - provider.start(); - - LOG.info("Service-account user refresher is {}", enabled ? "enabled" : "disabled"); - } catch (RuntimeException excp) { - LOG.error("Failed to initialize service-account user refresher; continuing with configured delegation users", excp); - - provider = new ServiceAccountUsersProvider(false, config.getAuthzProperties()); - } - - serviceAccountUsersProvider = provider; - } - private void startTomcat() throws Exception { tomcat = new Tomcat(); @@ -280,7 +255,7 @@ private void addAuthFilter(Context ctx) { * that it can be injected into {@link org.apache.ranger.pdp.rest.RangerPdpREST}. */ private void addJerseyServlet(Context ctx) { - RangerPdpApplication jerseyApp = new RangerPdpApplication(authorizer, config, serviceAccountUsersProvider); + RangerPdpApplication jerseyApp = new RangerPdpApplication(authorizer, config); Servlet jerseyServlet = new ServletContainer(jerseyApp); Tomcat.addServlet(ctx, "jerseyServlet", jerseyServlet).setLoadOnStartup(1); diff --git a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java index b9c9ba428d8..13f3497709b 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConfig.java @@ -52,9 +52,6 @@ public class RangerPdpConfig { private static final String DEFAULT_CONFIG_FILE = "ranger-pdp-default.xml"; private static final String SITE_CONFIG_FILE = "ranger-pdp-site.xml"; - private static final String DEFAULT_POLICY_REST_PREFIX = "ranger.authz.default.policy.rest."; - private static final String SERVICE_ACCOUNT_POLICY_REST_PREFIX = RangerPdpConstants.PROP_SERVICE_ACCOUNT_REFRESHER_PREFIX + "policy.rest."; - private final Properties props = new Properties(); public RangerPdpConfig() { @@ -194,26 +191,6 @@ public Properties getAuthzProperties() { return new Properties(props); } - public Properties getServiceAccountRefresherProperties() { - Properties ret = getAuthzProperties(); - - for (String name : props.stringPropertyNames()) { - if (name.startsWith(DEFAULT_POLICY_REST_PREFIX)) { - String targetName = SERVICE_ACCOUNT_POLICY_REST_PREFIX + name.substring(DEFAULT_POLICY_REST_PREFIX.length()); - - if (ret.getProperty(targetName) == null) { - ret.setProperty(targetName, props.getProperty(name)); - } - } - } - - return ret; - } - - public boolean isServiceAccountRefresherEnabled() { - return getBoolean(RangerPdpConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, true); - } - public String get(String key, String defaultValue) { String val = props.getProperty(key); diff --git a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java index 705d7253f45..d695aa7de1d 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/config/RangerPdpConstants.java @@ -87,9 +87,7 @@ private RangerPdpConstants() { public static final String PROP_AUTHZ_PREFIX = "ranger.authz."; public static final String PROP_PDP_PREFIX = "ranger.pdp."; public static final String PROP_PDP_SERVICE_PREFIX = PROP_PDP_PREFIX + "service."; - public static final String PROP_SERVICE_ACCOUNT_REFRESHER_PREFIX = "ranger.serviceaccount.refresher."; - public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_SERVICE_ACCOUNT_REFRESHER_PREFIX + "enabled"; - + public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_PDP_PREFIX + "serviceaccount.refresher.enabled"; // delegation users public static final String PROP_SUFFIX_DELEGATION_USERS = ".delegation.users"; public static final String WILDCARD_SERVICE_NAME = "*"; diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java new file mode 100644 index 00000000000..fe1948c5048 --- /dev/null +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrap.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.pdp.rest; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Starts {@link RangerPdpREST} service-account refresh from the JAX-RS / HK2 runtime + * (same lifecycle as delegated authorization checks), mirroring audit ingestor's bootstrap. + */ +@Singleton +public class PdpServiceAccountRefreshBootstrap { + private final RangerPdpREST rangerPdpREST; + + @Inject + public PdpServiceAccountRefreshBootstrap(RangerPdpREST rangerPdpREST) { + this.rangerPdpREST = rangerPdpREST; + } + + @PostConstruct + public void start() { + rangerPdpREST.startServiceAccountUsersRefresh(); + } + + @PreDestroy + public void stop() { + rangerPdpREST.stopServiceAccountUsersRefresh(); + } +} diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java index 64967d58381..ecaaee48fb4 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpApplication.java @@ -21,7 +21,6 @@ import org.apache.ranger.authz.api.RangerAuthorizer; import org.apache.ranger.pdp.config.RangerPdpConfig; -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.server.ResourceConfig; @@ -36,31 +35,30 @@ *
  • {@link AuthorizerBinder} – HK2 binder that makes the {@link RangerAuthorizer} * instance injectable into {@link RangerPdpREST} via {@code @Inject} *
  • {@link RangerPdpREST} – the JAX-RS resource class + *
  • {@link PdpServiceAccountRefreshBootstrap} – service-account refresh lifecycle * */ public class RangerPdpApplication extends ResourceConfig { - public RangerPdpApplication(RangerAuthorizer authorizer, RangerPdpConfig config, ServiceAccountUsersProvider serviceAccountUsersProvider) { + public RangerPdpApplication(RangerAuthorizer authorizer, RangerPdpConfig config) { register(JacksonFeature.class); - register(new AuthorizerBinder(authorizer, config, serviceAccountUsersProvider)); + register(new AuthorizerBinder(authorizer, config)); register(RangerPdpREST.class); + register(PdpServiceAccountRefreshBootstrap.class); } private static class AuthorizerBinder extends AbstractBinder { - private final RangerAuthorizer authorizer; - private final RangerPdpConfig config; - private final ServiceAccountUsersProvider serviceAccountUsersProvider; + private final RangerAuthorizer authorizer; + private final RangerPdpConfig config; - AuthorizerBinder(RangerAuthorizer authorizer, RangerPdpConfig config, ServiceAccountUsersProvider serviceAccountUsersProvider) { - this.authorizer = authorizer; - this.config = config; - this.serviceAccountUsersProvider = serviceAccountUsersProvider; + AuthorizerBinder(RangerAuthorizer authorizer, RangerPdpConfig config) { + this.authorizer = authorizer; + this.config = config; } @Override protected void configure() { bind(authorizer).to(RangerAuthorizer.class); bind(config).to(RangerPdpConfig.class); - bind(serviceAccountUsersProvider).to(ServiceAccountUsersProvider.class); } } } diff --git a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java index bbb8e464bd6..ae509627b04 100644 --- a/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java +++ b/pdp/src/main/java/org/apache/ranger/pdp/rest/RangerPdpREST.java @@ -22,8 +22,11 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authz.api.RangerAuthorizer; import org.apache.ranger.authz.api.RangerAuthzException; +import org.apache.ranger.authz.embedded.RangerAuthzConfig; import org.apache.ranger.authz.model.RangerAccessInfo; import org.apache.ranger.authz.model.RangerAuthzRequest; import org.apache.ranger.authz.model.RangerAuthzResult; @@ -39,7 +42,6 @@ import org.apache.ranger.pdp.config.RangerPdpConfig; import org.apache.ranger.pdp.config.RangerPdpConstants; import org.apache.ranger.pdp.model.ErrorResponse; -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,11 +59,15 @@ import javax.ws.rs.core.Response; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.stream.Collectors; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; @@ -71,6 +77,7 @@ import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static org.apache.ranger.authz.api.RangerAuthzApiErrorCode.INVALID_REQUEST_USER_INFO_MISSING; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_PDP_SERVICE_PREFIX; +import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SUFFIX_DELEGATION_USERS; import static org.apache.ranger.pdp.config.RangerPdpConstants.WILDCARD_SERVICE_NAME; @@ -102,7 +109,15 @@ public class RangerPdpREST { private static final Response RESPONSE_OK = Response.ok().build(); - private final Map> delegationUsersByService = new HashMap<>(); + private static final String PROP_POLICY_REST_URL = RangerAuthzConfig.PROP_PREFIX_DEFAULT + "policy.rest.url"; + private static final String PROP_POLICY_POLL_INTERVAL_MS = RangerAuthzConfig.PROP_PREFIX_DEFAULT + "policy.pollIntervalMs"; + private static final long DEFAULT_POLL_INTERVAL_MS = 30 * 1000L; + + private final Map> configuredDelegationUsers = new HashMap<>(); + + private volatile Map> delegationUsersByService = Collections.emptyMap(); + private RangerAdminRESTClient rangerAdminClient; + private Timer serviceAccountUsersRefreshTimer; @Inject private RangerAuthorizer authorizer; @@ -110,15 +125,29 @@ public class RangerPdpREST { @Inject private RangerPdpConfig config; - @Inject - private ServiceAccountUsersProvider serviceAccountUsersProvider; - @Context private ServletContext servletContext; @PostConstruct public void initialize() { - initializeDelegationUsers(); + initializeDelegationUsersOnly(); + } + + /** + * Starts Admin {@code service.account.users} polling. Invoked from {@link PdpServiceAccountRefreshBootstrap} + * after {@link #initialize()} has loaded static delegation config. + */ + public synchronized void startServiceAccountUsersRefresh() { + Properties properties = config != null ? config.getAuthzProperties() : new Properties(); + startServiceAccountUsersRefresh(properties); + } + + public synchronized void stopServiceAccountUsersRefresh() { + Timer timer = serviceAccountUsersRefreshTimer; + serviceAccountUsersRefreshTimer = null; + if (timer != null) { + timer.cancel(); + } } /** @@ -469,28 +498,29 @@ private void recordRequestMetrics(Response ret, long startNanos, HttpServletRequ } private boolean isDelegationUserForService(String serviceName, String userName) { - boolean ret = false; + final boolean ret; + Map> currentDelegationUsersByService = delegationUsersByService; - if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) { - Set delegationUsers = delegationUsersByService.get(serviceName); + if (currentDelegationUsersByService.isEmpty()) { + ret = false; + } else if (StringUtils.isBlank(serviceName) || StringUtils.isBlank(userName)) { + ret = false; + } else { + Set delegationUsers = currentDelegationUsersByService.get(serviceName); if (delegationUsers == null) { - delegationUsers = delegationUsersByService.get(RangerPdpConstants.WILDCARD_SERVICE_NAME); + delegationUsers = currentDelegationUsersByService.get(RangerPdpConstants.WILDCARD_SERVICE_NAME); } ret = delegationUsers != null && delegationUsers.contains(userName); - - if (!ret && serviceAccountUsersProvider != null) { - ret = serviceAccountUsersProvider.isServiceAccountUser(serviceName, userName); - } } - LOG.debug("isDelegationUserForService(serviceName={}): ret={}", serviceName, ret); + LOG.debug("isDelegationUserForService(serviceName={}, userName={}): ret={}", serviceName, userName, ret); return ret; } - private void initializeDelegationUsers() { + private void initializeDelegationUsersOnly() { Properties properties = config != null ? config.getAuthzProperties() : new Properties(); for (String key : properties.stringPropertyNames()) { @@ -507,21 +537,118 @@ private void initializeDelegationUsers() { .collect(Collectors.toSet()); if (!delegationUsers.isEmpty()) { - delegationUsersByService.put(serviceName, delegationUsers); + configuredDelegationUsers.put(serviceName, delegationUsers); - LOG.info("Configured {} delegation users for service '{}'", delegationUsers.size(), serviceName); + LOG.info("Delegation users for service '{}': {}", serviceName, delegationUsers); } } } - if (delegationUsersByService.isEmpty()) { + delegationUsersByService = mergeDelegationUsers(configuredDelegationUsers, Collections.emptyMap()); + + if (configuredDelegationUsers.isEmpty()) { LOG.warn("No delegation users configured"); + } + } + + void refreshDelegationUsers() { + RangerAdminRESTClient currentRangerAdminClient = rangerAdminClient; + + if (currentRangerAdminClient != null) { + try { + delegationUsersByService = mergeDelegationUsers(configuredDelegationUsers, currentRangerAdminClient.getServiceAccountUsers()); + } catch (Exception e) { + LOG.warn("Failed to refresh service account users from Ranger Admin", e); + } } else { - Set wildcardDelegationUsers = delegationUsersByService.get(WILDCARD_SERVICE_NAME); + LOG.warn("Failed to refresh service account users from Ranger admin: admin client not initialized"); + } + } + + static Map> mergeDelegationUsers(Map> configuredDelegationUsers, Map> serviceAccountUsersByServiceName) { + Map> ret = new HashMap<>(); + Set wildcardDelegationUsers = configuredDelegationUsers.get(WILDCARD_SERVICE_NAME); + + if (wildcardDelegationUsers == null) { + wildcardDelegationUsers = Collections.emptySet(); + } + + for (Map.Entry> entry : configuredDelegationUsers.entrySet()) { + Set delegationUsers = new HashSet<>(entry.getValue()); + + if (!WILDCARD_SERVICE_NAME.equals(entry.getKey())) { + delegationUsers.addAll(wildcardDelegationUsers); + } + + ret.put(entry.getKey(), delegationUsers); + } + + for (Map.Entry> entry : serviceAccountUsersByServiceName.entrySet()) { + String serviceName = entry.getKey(); + Set serviceAccountUsers = entry.getValue(); + + if (StringUtils.isNotBlank(serviceName) && serviceAccountUsers != null && !serviceAccountUsers.isEmpty()) { + Set delegationUsersForService = ret.get(serviceName); + Set delegationUsers = new HashSet<>(); + + if (delegationUsersForService != null) { + delegationUsers.addAll(delegationUsersForService); + } else if (!WILDCARD_SERVICE_NAME.equals(serviceName)) { + delegationUsers.addAll(wildcardDelegationUsers); + } + + delegationUsers.addAll(serviceAccountUsers); + ret.put(serviceName, delegationUsers); + } + } + + return ret; + } + + private void startServiceAccountUsersRefresh(Properties properties) { + if (!Boolean.parseBoolean(StringUtils.defaultString(properties.getProperty(PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED), "false"))) { + LOG.info("service account user refresh is not enabled. Property: {}", PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED); + return; + } + + LOG.info("Starting service account user refresher"); + + String rangerAdminUrl = properties.getProperty(PROP_POLICY_REST_URL); + + if (serviceAccountUsersRefreshTimer == null && StringUtils.isNotBlank(rangerAdminUrl)) { + try { + Configuration rangerAdminConfig = new Configuration(false); + RangerAdminRESTClient client = new RangerAdminRESTClient(); + + for (String key : properties.stringPropertyNames()) { + String value = properties.getProperty(key); + + if (value != null) { + rangerAdminConfig.set(key, value); + } + } + + client.init("ranger-pdp", "ranger-pdp", StringUtils.removeEnd(RangerAuthzConfig.PROP_PREFIX_DEFAULT, "."), rangerAdminConfig); + + rangerAdminClient = client; + + long pollIntervalMs = rangerAdminConfig.getLong(PROP_POLICY_POLL_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS); + Timer timer = new Timer("pdpServiceAccountUsersRefresh", true); + + refreshDelegationUsers(); + + timer.schedule(new TimerTask() { + @Override + public void run() { + refreshDelegationUsers(); + } + }, pollIntervalMs, pollIntervalMs); + + serviceAccountUsersRefreshTimer = timer; - // delegation users for WILDCARD_SERVICE_NAME are delegates for all services - if (wildcardDelegationUsers != null && !wildcardDelegationUsers.isEmpty()) { - delegationUsersByService.forEach((k, v) -> v.addAll(wildcardDelegationUsers)); + LOG.info("Scheduled service account user refresh every {} ms from {}", pollIntervalMs, rangerAdminUrl); + } catch (Exception e) { + LOG.warn("Failed to start service account user refresh", e); } } } diff --git a/pdp/src/main/resources/ranger-pdp-default.xml b/pdp/src/main/resources/ranger-pdp-default.xml index f1c7334c3df..88ac3a6ad68 100644 --- a/pdp/src/main/resources/ranger-pdp-default.xml +++ b/pdp/src/main/resources/ranger-pdp-default.xml @@ -230,12 +230,20 @@ Policy source implementation. Use RangerAdminRESTClient to download policies live from Ranger Admin. + + ranger.pdp.serviceaccount.refresher.enabled + false + When true and ranger.authz.default.policy.rest.url is set, periodically merge service.account.users from Ranger Admin into PDP delegation users. Override via Cloudera Manager safety valve. Poll credentials use ranger.authz.default.policy.rest.* (same as policy download). + + ranger.authz.default.policy.rest.url http://localhost:6080 URL of the Ranger Admin server. Comma-separated for HA. Example: http://ranger-admin-1:6080,http://ranger-admin-2:6080 + Also used for service-account user refresh when ranger.pdp.serviceaccount.refresher.enabled is true (rest client auth via ranger.authz.default.policy.rest.ssl.config.file and .client.username/.password). + The REST client identity must be listed in ranger.admin.service.account.info.viewers on Admin. @@ -268,19 +276,7 @@ ranger.authz.default.policy.pollIntervalMs 30000 - How often (ms) to poll Ranger Admin for updated policies and roles. - - - - ranger.serviceaccount.refresher.enabled - true - Enable periodic retrieval of service-account users from Ranger Admin. - - - - ranger.serviceaccount.refresher.pollIntervalMs - 30000 - How often (ms) to refresh service-account users from Ranger Admin. + How often (ms) to poll Ranger Admin for updated policies and roles and to refresh service-account users when ranger.pdp.serviceaccount.refresher.enabled is true. diff --git a/pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java b/pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java new file mode 100644 index 00000000000..8b0f617bd98 --- /dev/null +++ b/pdp/src/test/java/org/apache/ranger/pdp/rest/PdpServiceAccountRefreshBootstrapTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.pdp.rest; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PdpServiceAccountRefreshBootstrapTest { + @Test + void startStopDelegateToRangerPdpREST() { + TrackingPdpRest rest = new TrackingPdpRest(); + PdpServiceAccountRefreshBootstrap bootstrap = new PdpServiceAccountRefreshBootstrap(rest); + + bootstrap.start(); + bootstrap.stop(); + + assertTrue(rest.startCalled); + assertTrue(rest.stopCalled); + } + + private static final class TrackingPdpRest extends RangerPdpREST { + boolean startCalled; + boolean stopCalled; + + @Override + public synchronized void startServiceAccountUsersRefresh() { + startCalled = true; + } + + @Override + public synchronized void stopServiceAccountUsersRefresh() { + stopCalled = true; + } + } +} diff --git a/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java b/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java index da894aefd4f..5832a58796f 100644 --- a/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java +++ b/pdp/src/test/java/org/apache/ranger/pdp/rest/RangerPdpRESTTest.java @@ -19,6 +19,7 @@ package org.apache.ranger.pdp.rest; +import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authz.api.RangerAuthorizer; import org.apache.ranger.authz.api.RangerAuthzApiErrorCode; import org.apache.ranger.authz.api.RangerAuthzException; @@ -34,7 +35,6 @@ import org.apache.ranger.pdp.RangerPdpStats; import org.apache.ranger.pdp.config.RangerPdpConfig; import org.apache.ranger.pdp.config.RangerPdpConstants; -import org.apache.ranger.plugin.util.ServiceAccountUsersProvider; import org.junit.jupiter.api.Test; import javax.servlet.ServletContext; @@ -46,6 +46,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -53,6 +54,7 @@ import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_PDP_SERVICE_PREFIX; import static org.apache.ranger.pdp.config.RangerPdpConstants.PROP_SUFFIX_DELEGATION_USERS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; public class RangerPdpRESTTest { private static final String CALLER_DELEGATION_ALLOWED = "alice"; @@ -207,6 +209,62 @@ public void testGetResourcePermissionsReturnsBadRequestOnAuthzException() throws assertEquals(1L, stats.getTotalAuthzBadRequest()); } + @Test + public void testMergeDelegationUsersUnionsServiceAccountUsersAndWildcard() { + Map> configuredDelegationUsers = new HashMap<>(); + + configuredDelegationUsers.put("*", new HashSet<>(Collections.singleton("wild"))); + configuredDelegationUsers.put("svc1", new HashSet<>(Collections.singleton(CALLER_DELEGATION_ALLOWED))); + + Map> serviceAccountUsersByServiceName = new HashMap<>(); + + serviceAccountUsersByServiceName.put("svc1", Collections.singleton("svc-user")); + serviceAccountUsersByServiceName.put("svc2", Collections.singleton("only-dynamic")); + + Map> mergedDelegationUsers = RangerPdpREST.mergeDelegationUsers(configuredDelegationUsers, serviceAccountUsersByServiceName); + + assertEquals(new HashSet<>(Arrays.asList(CALLER_DELEGATION_ALLOWED, "wild", "svc-user")), mergedDelegationUsers.get("svc1")); + assertEquals(new HashSet<>(Arrays.asList("wild", "only-dynamic")), mergedDelegationUsers.get("svc2")); + assertEquals(Collections.singleton("wild"), mergedDelegationUsers.get("*")); + } + + @Test + public void testRefreshKeepsDelegationUsersWhenAdminCallFails() throws Exception { + RangerPdpREST rest = createRest(new TestAuthorizer(), new RangerPdpStats()); + Map> before = delegationUsers(rest); + + setField(rest, "rangerAdminClient", new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() throws Exception { + throw new Exception("admin down"); + } + }); + + rest.refreshDelegationUsers(); + + assertSame(before, delegationUsers(rest)); + } + + @Test + public void testRefreshAddsServiceAccountUsers() throws Exception { + RangerPdpStats stats = new RangerPdpStats(); + RangerPdpREST rest = createRest(new TestAuthorizer(new RangerAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW)), stats); + RangerAuthzRequest request = new RangerAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1); + + setField(rest, "rangerAdminClient", new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.singletonMap("svc1", Collections.singleton("svc-user")); + } + }); + + rest.refreshDelegationUsers(); + + Response response = rest.authorize(request, httpRequest("svc-user", stats)); + + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + } + @Test public void testServiceAccountUserCanCallAllPdpApis() throws Exception { RangerPdpStats stats = new RangerPdpStats(); @@ -214,45 +272,49 @@ public void testServiceAccountUserCanCallAllPdpApis() throws Exception { new RangerAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW), new RangerMultiAuthzResult("req-1", RangerAuthzResult.AccessDecision.ALLOW), new RangerResourcePermissions(), null); - RangerPdpREST rest = createRest(authorizer, stats, serviceAccountProvider(CALLER_DELEGATION_NOT_ALLOWED)); + RangerPdpREST rest = createRest(authorizer, stats); + String dynamicCaller = "svc-user"; + + setField(rest, "rangerAdminClient", new RangerAdminRESTClient() { + @Override + public Map> getServiceAccountUsers() { + return Collections.singletonMap("svc1", Collections.singleton(dynamicCaller)); + } + }); + rest.refreshDelegationUsers(); assertEquals(Response.Status.OK.getStatusCode(), rest.authorize( new RangerAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1), - httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + httpRequest(dynamicCaller, stats)).getStatus()); assertEquals(Response.Status.OK.getStatusCode(), rest.authorizeMulti( new RangerMultiAuthzRequest("req-1", USER_USER1, null, CONTEXT_SERVICE_SVC1), - httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + httpRequest(dynamicCaller, stats)).getStatus()); assertEquals(Response.Status.OK.getStatusCode(), rest.getResourcePermissions( new RangerResourcePermissionsRequest("req-1", null, CONTEXT_SERVICE_SVC1), - httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + httpRequest(dynamicCaller, stats)).getStatus()); assertEquals(Response.Status.OK.getStatusCode(), rest.filterResources( new RangerFilterResourcesRequest("user1", Arrays.asList("table:default/t1"), "_any", "hive", "svc1"), - httpRequest(CALLER_DELEGATION_NOT_ALLOWED, stats)).getStatus()); + httpRequest(dynamicCaller, stats)).getStatus()); } private static RangerPdpREST createRest(TestAuthorizer authorizer, RangerPdpStats stats) throws Exception { - return createRest(authorizer, stats, null); - } - - private static RangerPdpREST createRest(TestAuthorizer authorizer, RangerPdpStats stats, ServiceAccountUsersProvider provider) throws Exception { RangerPdpREST rest = new RangerPdpREST(); setField(rest, "authorizer", authorizer); setField(rest, "config", new TestConfig(defaultConfig())); - setField(rest, "serviceAccountUsersProvider", provider); setField(rest, "servletContext", servletContextWithStats(stats)); rest.initialize(); return rest; } - private static ServiceAccountUsersProvider serviceAccountProvider(String user) throws Exception { - ServiceAccountUsersProvider provider = new ServiceAccountUsersProvider(false, new Properties()); - Map> users = Collections.singletonMap("svc1", Collections.singleton(user)); + @SuppressWarnings("unchecked") + private static Map> delegationUsers(RangerPdpREST rest) throws Exception { + Field field = RangerPdpREST.class.getDeclaredField("delegationUsersByService"); - setField(provider, "serviceAccountUsers", users); + field.setAccessible(true); - return provider; + return (Map>) field.get(rest); } private static HttpServletRequest httpRequest(String user, RangerPdpStats stats) { diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 2ca3d8f0513..2a6e74eff78 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -216,8 +216,8 @@ ranger.admin.service.account.info.viewers ranger-pdp,ranger-audit-ingestor,admin,rangerauditserver - Comma-separated users allowed to view service-account users and groups in service-header responses. - Additional users can be appended to this list. + Comma-separated logins (and Kerberos short names) allowed to receive service.account.users and service.account.groups on GET /public/v2/api/service-headers. + Must include identities used by ranger.pdp.serviceaccount.refresher.enabled / ranger.audit.ingestor.serviceaccount.refresher.enabled (ranger.authz.default.policy.rest.* and ranger.audit.ingestor.policy.rest.* clients). Safety-valve append on this list. From 08462a335dd2cdf72a33848e4af56f0386b16032 Mon Sep 17 00:00:00 2001 From: ramk Date: Wed, 18 Nov 2026 13:33:20 +0530 Subject: [PATCH 4/4] RANGER-5869: Drop redundant PDP assembly dependencySet The extra dependencySet never matched audit jars in CI and failed the assembly-warnings gate; audit modules are already in the moduleSet. --- distro/src/main/assembly/pdp.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/distro/src/main/assembly/pdp.xml b/distro/src/main/assembly/pdp.xml index cd9b2e5938b..37fad91fa4b 100644 --- a/distro/src/main/assembly/pdp.xml +++ b/distro/src/main/assembly/pdp.xml @@ -146,19 +146,6 @@ - - - - lib - false - false - - org.apache.ranger:ranger-audit-core:jar:${project.version} - org.apache.ranger:ranger-audit-dest-auditserver:jar:${project.version} - - - -