Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.plugin.model.RangerRole;
import org.apache.ranger.plugin.model.RangerServiceHeaderInfo;
import org.apache.ranger.plugin.util.GrantRevokeRequest;
import org.apache.ranger.plugin.util.GrantRevokeRoleRequest;
import org.apache.ranger.plugin.util.JsonUtilsV2;
Expand All @@ -51,15 +52,18 @@

import java.io.UnsupportedEncodingException;
import java.security.PrivilegedExceptionAction;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;

public class RangerAdminRESTClient extends AbstractRangerAdminClient {
private static final Logger LOG = LoggerFactory.getLogger(RangerAdminRESTClient.class);

private static final TypeReference<List<String>> TYPE_LIST_STRING = new TypeReference<List<String>>() {};
private static final TypeReference<List<String>> TYPE_LIST_STRING = new TypeReference<List<String>>() {};
private static final TypeReference<List<RangerServiceHeaderInfo>> TYPE_LIST_SERVICE_HEADER = new TypeReference<List<RangerServiceHeaderInfo>>() {};

private final String pluginCapabilities = Long.toHexString(new RangerPluginCapability().getPluginCapabilities());
private final RangerRESTUtils restUtils = new RangerRESTUtils();
Expand Down Expand Up @@ -116,7 +120,7 @@ public void init(String serviceName, String appId, String propertyPrefix, Config
url = url.substring(0, url.length() - 1);
}

init(url, sslConfigFileName, restClientConnTimeOutMs, restClientReadTimeOutMs, restClientMaxRetryAttempts, restClientRetryIntervalMs, config);
init(url, sslConfigFileName, restClientConnTimeOutMs, restClientReadTimeOutMs, restClientMaxRetryAttempts, restClientRetryIntervalMs, config, propertyPrefix);

try {
this.serviceNameUrlParam = URLEncoderUtil.encodeURIParam(serviceName);
Expand All @@ -126,6 +130,59 @@ public void init(String serviceName, String appId, String propertyPrefix, Config
}
}

public Map<String, Set<String>> getServiceAccountUsers() throws Exception {
LOG.debug("==> RangerAdminRESTClient.getServiceAccountUsers()");

final boolean isSecureMode = isAuthenticationEnabled();
final Cookie sessionId = this.sessionId;
final Map<String, String> queryParams = Collections.emptyMap();
final Response response;

if (isSecureMode) {
response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<Response>) () ->
restClient.get(RangerRESTUtils.REST_URL_SERVICE_HEADERS, queryParams, sessionId));
} else {
response = restClient.get(RangerRESTUtils.REST_URL_SERVICE_HEADERS, queryParams, sessionId);
}

checkAndResetSessionCookie(response);

Map<String, Set<String>> ret = Collections.emptyMap();

if (response == null) {
throw new Exception("Unknown error while retrieving service-account users");
} else if (response.getStatus() == HttpStatus.SC_OK) {
List<RangerServiceHeaderInfo> serviceHeaders = JsonUtilsV2.jsonToObj(response.readEntity(String.class),
TYPE_LIST_SERVICE_HEADER);

if (serviceHeaders != null && !serviceHeaders.isEmpty()) {
Map<String, Set<String>> serviceAccountUsers = new HashMap<>();

for (RangerServiceHeaderInfo serviceHeader : serviceHeaders) {
if (serviceHeader != null && Boolean.TRUE.equals(serviceHeader.getIsEnabled()) &&
!StringUtil.isEmpty(serviceHeader.getName()) && serviceHeader.getServiceAccountUsers() != null &&
!serviceHeader.getServiceAccountUsers().isEmpty()) {
serviceAccountUsers.put(serviceHeader.getName(), serviceHeader.getServiceAccountUsers());
}
}

ret = serviceAccountUsers;
}
} else {
LOG.error("getServiceAccountUsers() failed: HTTP status={}, isSecure={}", response.getStatus(), isSecureMode);

if (response.getStatus() == HttpStatus.SC_UNAUTHORIZED || response.getStatus() == HttpStatus.SC_FORBIDDEN) {
throw new AccessControlException();
}

throw new Exception("HTTP " + response.getStatus() + " while retrieving service-account users");
}

LOG.debug("<== RangerAdminRESTClient.getServiceAccountUsers(): serviceCount={}", ret.size());

return ret;
}

@Override
public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception {
LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated({}, {})", lastKnownVersion, lastActivationTimeInMillis);
Expand Down Expand Up @@ -1029,10 +1086,10 @@ public void setTokenSupplier(Supplier<String> tokenSupplier) {
}
}

private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs, int restClientReadTimeOutMs, int restClientMaxRetryAttempts, int restClientRetryIntervalMs, Configuration config) {
private void init(String url, String sslConfigFileName, int restClientConnTimeOutMs, int restClientReadTimeOutMs, int restClientMaxRetryAttempts, int restClientRetryIntervalMs, Configuration config, String propertyPrefix) {
LOG.debug("==> RangerAdminRESTClient.init({}, {})", url, sslConfigFileName);

restClient = new RangerRESTClient(url, sslConfigFileName, config);
restClient = new RangerRESTClient(url, sslConfigFileName, config, propertyPrefix);
if (tokenSupplier != null) {
restClient.setTokenSupplier(tokenSupplier);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public class RangerRESTUtils {
public static final String REST_URL_SERVICE_GET_ROLE_INFO = "/service/public/v2/api/roles/name/";
public static final String REST_URL_SERVICE_GRANT_ROLE = "/service/public/v2/api/roles/grant/";
public static final String REST_URL_SERVICE_REVOKE_ROLE = "/service/public/v2/api/roles/revoke/";
public static final String REST_URL_SERVICE_HEADERS = "/service/public/v2/api/service-headers";
public static final String REST_URL_SERVICE_SERCURE_GET_USER_GROUP_ROLES = "/service/roles/secure/download/";
public static final String REST_URL_SERVICE_GET_USER_GROUP_ROLES = "/service/roles/download/";
public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download/";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.ranger.plugin.util.GrantRevokeRequest;
import org.apache.ranger.plugin.util.GrantRevokeRoleRequest;
import org.apache.ranger.plugin.util.RangerRESTClient;
import org.apache.ranger.plugin.util.RangerRESTUtils;
import org.apache.ranger.plugin.util.RangerRoles;
import org.apache.ranger.plugin.util.RangerServiceNotFoundException;
import org.apache.ranger.plugin.util.ServiceGdsInfo;
Expand All @@ -51,8 +52,10 @@
import java.lang.reflect.Field;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* @generated by Cursor
Expand Down Expand Up @@ -1146,4 +1149,45 @@ public void test42_secureEndpointAccessDisabled_withoutKerberos_usesNonSecureUrl
misc.verify(() -> MiscUtil.executePrivilegedAction(Mockito.<PrivilegedExceptionAction<Response>>any()), Mockito.never());
}
}

@Test
public void test43_getServiceAccountUsers() throws Exception {
RangerAdminRESTClient client = new RangerAdminRESTClient();
Configuration cfg = new Configuration(false);
cfg.set("p.policy.rest.url", "http://localhost:6080");
cfg.setBoolean("p.forceSecureEndpointAccess", false);
client.init("svc", "app", "p", cfg);
RangerRESTClient rest = Mockito.mock(RangerRESTClient.class);
setPrivateField(client, "restClient", rest);

Response response = mockResponse(200,
"[{\"name\":\"svc1\",\"isEnabled\":true,\"serviceAccountUsers\":[\"alice\"]}," +
"{\"name\":\"svc2\",\"isEnabled\":false,\"serviceAccountUsers\":[\"bob\"]}," +
"{\"name\":\"svc3\",\"isEnabled\":true}]",
Collections.emptyList());
Mockito.when(rest.get(Mockito.anyString(), Mockito.anyMap(), Mockito.any())).thenReturn(response);

Map<String, Set<String>> users = client.getServiceAccountUsers();

Assertions.assertEquals(Collections.singleton("alice"), users.get("svc1"));
Assertions.assertEquals(1, users.size());
Mockito.verify(rest).get(Mockito.eq(RangerRESTUtils.REST_URL_SERVICE_HEADERS), Mockito.anyMap(), Mockito.any());
}

@Test
public void test44_getServiceAccountUsers_forbidden() throws Exception {
RangerAdminRESTClient client = new RangerAdminRESTClient();
Configuration cfg = new Configuration(false);
cfg.set("p.policy.rest.url", "http://localhost:6080");
cfg.setBoolean("p.forceSecureEndpointAccess", false);
client.init("svc", "app", "p", cfg);
RangerRESTClient rest = Mockito.mock(RangerRESTClient.class);
setPrivateField(client, "restClient", rest);
Response response = Mockito.mock(Response.class);
Mockito.when(response.getStatus()).thenReturn(403);
Mockito.when(rest.get(Mockito.anyString(), Mockito.anyMap(), Mockito.any()))
.thenReturn(response);

Assertions.assertThrows(AccessControlException.class, client::getServiceAccountUsers);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ private AuditServerConstants() {}
public static final String PROP_PREFIX_AUDIT_SERVER_SERVICE = PROP_PREFIX_AUDIT_SERVER + "service.";
public static final String PROP_AUTH_TO_LOCAL = PROP_PREFIX_AUDIT_SERVER + "auth.to.local";
public static final String PROP_SUFFIX_ALLOWED_USERS = ".allowed.users";
public static final String PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED = PROP_PREFIX_AUDIT_SERVER + "serviceaccount.refresher.enabled";
public static final String PROP_AUDIT_SERVICE_PRINCIPAL = "service.kerberos.principal";
public static final String PROP_AUDIT_SERVICE_KEYTAB = "service.kerberos.keytab";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
package org.apache.ranger.audit.rest;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.ranger.admin.client.RangerAdminRESTClient;
import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.apache.ranger.audit.producer.AuditDestinationMgr;
import org.apache.ranger.audit.provider.MiscUtil;
Expand All @@ -43,10 +45,14 @@
import javax.ws.rs.core.Response;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;

import static org.apache.ranger.audit.server.AuditServerConstants.PROP_PREFIX_AUDIT_SERVER_SERVICE;
Expand All @@ -58,13 +64,109 @@
public class AuditREST {
private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class);

private static final Map<String, Set<String>> allowedServiceUsers;
private static final String RANGER_ADMIN_CONFIG_PREFIX = "ranger.audit.ingestor";
private static final String PROP_POLICY_REST_URL = RANGER_ADMIN_CONFIG_PREFIX + ".policy.rest.url";
private static final String PROP_POLICY_POLL_INTERVAL_MS = RANGER_ADMIN_CONFIG_PREFIX + ".policy.pollIntervalMs";
private static final long DEFAULT_POLL_INTERVAL_MS = 30 * 1000L;

private static final Map<String, Set<String>> configuredAllowedUsers;

private static volatile Map<String, Set<String>> allowedServiceUsers;
private static RangerAdminRESTClient rangerAdminClient;
private static Timer serviceAccountUsersRefreshTimer;

static {
allowedServiceUsers = initializeAllowedUsers();
configuredAllowedUsers = initializeAllowedUsers();
allowedServiceUsers = mergeAllowedUsers(configuredAllowedUsers, Collections.emptyMap());
initializeAuthToLocal();
}

public static synchronized void startServiceAccountUsersRefresh(Configuration configuration) {
if (!configuration.getBoolean(AuditServerConstants.PROP_SERVICE_ACCOUNT_REFRESHER_ENABLED, false)) {
return;
}

String rangerAdminUrl = configuration.get(PROP_POLICY_REST_URL);

if (serviceAccountUsersRefreshTimer == null && StringUtils.isNotBlank(rangerAdminUrl)) {
try {
RangerAdminRESTClient client = new RangerAdminRESTClient();

client.init("audit-ingestor", "ranger-audit", RANGER_ADMIN_CONFIG_PREFIX, configuration);

rangerAdminClient = client;

long pollIntervalMs = configuration.getLong(PROP_POLICY_POLL_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS);
Timer timer = new Timer("auditServiceAccountUsersRefresh", true);

timer.schedule(new TimerTask() {
@Override
public void run() {
refreshAllowedUsers(rangerAdminClient);
}
}, pollIntervalMs, pollIntervalMs);

serviceAccountUsersRefreshTimer = timer;

LOG.info("Scheduled service account user refresh every {} ms from {}", pollIntervalMs, rangerAdminUrl);
refreshAllowedUsers(client);
} catch (Exception e) {
LOG.warn("Failed to start service account user refresh", e);
}
}
}

public static synchronized void stopServiceAccountUsersRefresh() {
Timer timer = serviceAccountUsersRefreshTimer;

serviceAccountUsersRefreshTimer = null;

if (timer != null) {
timer.cancel();
}
}

static void refreshAllowedUsers(RangerAdminRESTClient client) {
if (client != null) {
try {
Map<String, Set<String>> serviceAccountUsers = client.getServiceAccountUsers();
allowedServiceUsers = mergeAllowedUsers(configuredAllowedUsers, serviceAccountUsers);
if (!serviceAccountUsers.isEmpty()) {
LOG.info("Merged service.account.users for {} service(s) from Ranger Admin", serviceAccountUsers.size());
}
} catch (Exception e) {
LOG.warn("Failed to refresh service account users from Ranger Admin", e);
}
}
}

static Map<String, Set<String>> mergeAllowedUsers(Map<String, Set<String>> configuredAllowedUsers, Map<String, Set<String>> serviceAccountUsersByServiceName) {
Map<String, Set<String>> ret = new HashMap<>();

for (Map.Entry<String, Set<String>> entry : configuredAllowedUsers.entrySet()) {
ret.put(entry.getKey(), new HashSet<>(entry.getValue()));
}

for (Map.Entry<String, Set<String>> entry : serviceAccountUsersByServiceName.entrySet()) {
String serviceName = entry.getKey();
Set<String> serviceAccountUsers = entry.getValue();

if (StringUtils.isNotBlank(serviceName) && serviceAccountUsers != null && !serviceAccountUsers.isEmpty()) {
Set<String> allowedUsers = ret.get(serviceName);

if (allowedUsers == null) {
allowedUsers = new HashSet<>();

ret.put(serviceName, allowedUsers);
}

allowedUsers.addAll(serviceAccountUsers);
}
}

return ret;
}

@Autowired
AuditDestinationMgr auditDestinationMgr;

Expand Down Expand Up @@ -285,6 +387,10 @@ private String applyAuthToLocal(String principal) {
return principal;
}

if (principal.startsWith("spiffe://")) {
return principal;
}

// Check if this looks like a Kerberos principal (has @ or /)
if (!principal.contains("@") && !principal.contains("/")) {
LOG.debug("Username '{}' is already a short name (JWT/basic auth), no auth_to_local mapping needed", principal);
Expand Down Expand Up @@ -342,10 +448,6 @@ private boolean isAllowedServiceUser(String serviceName, String userName) {
return ret;
}

/**
* Initialize the set of allowed service users from configuration
* @return Map of allowed usernames per service
*/
private static Map<String, Set<String>> initializeAllowedUsers() {
Map<String, Set<String>> ret = new HashMap<>();
AuditServerConfig config = AuditServerConfig.getInstance();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ranger.audit.rest;

import org.apache.ranger.audit.server.AuditServerConfig;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

/**
* Starts service-account refresh from the Spring webapp class loader so {@link AuditREST}
* static allow maps stay in sync with the JAX-RS beans that authorize ingest traffic.
*/
@Component
public class AuditServiceAccountRefreshBootstrap {
@PostConstruct
public void start() {
AuditREST.startServiceAccountUsersRefresh(AuditServerConfig.getInstance());
}

@PreDestroy
public void stop() {
AuditREST.stopServiceAccountUsersRefresh();
}
}
Loading