From 87265b9f876adbdecc665c2371b1a0eeddc33eb8 Mon Sep 17 00:00:00 2001 From: "huafeng.shi" Date: Sat, 5 Sep 2026 17:07:56 +0800 Subject: [PATCH] RANGER-5803: Add Apache Drill service definition for Ranger --- .../plugin/store/EmbeddedServiceDefsUtil.java | 7 +- .../service-defs/ranger-servicedef-drill.json | 140 +++++ distro/pom.xml | 6 + distro/src/main/assembly/admin-web.xml | 17 + plugin-drill/pom.xml | 89 +++ .../ranger/services/RangerServiceDrill.java | 565 ++++++++++++++++++ .../services/RangerServiceDrillTest.java | 237 ++++++++ pom.xml | 17 + 8 files changed, 1077 insertions(+), 1 deletion(-) create mode 100644 agents-common/src/main/resources/service-defs/ranger-servicedef-drill.json create mode 100644 plugin-drill/pom.xml create mode 100644 plugin-drill/src/main/java/org/apache/ranger/services/RangerServiceDrill.java create mode 100644 plugin-drill/src/test/java/org/apache/ranger/services/RangerServiceDrillTest.java diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index e2fe3fd9769..855fd5b09ba 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -48,13 +48,14 @@ public class EmbeddedServiceDefsUtil { // following servicedef list should be reviewed/updated whenever a new embedded service-def is added - public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,trino,ozone,kudu,schema-registry,nestedstructure,polaris"; + public static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,drill,kms,knox,storm,yarn,kafka,solr,atlas,nifi,nifi-registry,sqoop,kylin,elasticsearch,presto,trino,ozone,kudu,schema-registry,nestedstructure,polaris"; private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; + public static final String EMBEDDED_SERVICEDEF_DRILL_NAME = "drill"; public static final String EMBEDDED_SERVICEDEF_KMS_NAME = "kms"; public static final String EMBEDDED_SERVICEDEF_KNOX_NAME = "knox"; public static final String EMBEDDED_SERVICEDEF_STORM_NAME = "storm"; @@ -119,6 +120,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef elasticsearchServiceDef; private RangerServiceDef prestoServiceDef; private RangerServiceDef trinoServiceDef; + private RangerServiceDef drillServiceDef; private RangerServiceDef ozoneServiceDef; private RangerServiceDef kuduServiceDef; private RangerServiceDef nestedStructureServiveDef; @@ -168,6 +170,7 @@ public void init(ServiceStore store) { abfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ABFS_NAME); elasticsearchServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_ELASTICSEARCH_NAME); trinoServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TRINO_NAME); + drillServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_DRILL_NAME); prestoServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_PRESTO_NAME); ozoneServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_OZONE_NAME); kuduServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KUDU_NAME); @@ -251,6 +254,8 @@ public long getElasticsearchServiceDefId() { public long getTrinoServiceDefId() { return getId(trinoServiceDef); } + public long getDrillServiceDefId() { return getId(drillServiceDef); } + public long getPrestoServiceDefId() { return getId(prestoServiceDef); } public long getOzoneServiceDefId() { return getId(ozoneServiceDef); } diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-drill.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-drill.json new file mode 100644 index 00000000000..998327fbb5a --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-drill.json @@ -0,0 +1,140 @@ +{ + "name": "drill", + "label": "Apache Drill", + "description": "Apache Drill distributed SQL query engine", + "guid": "d8b1c0a2-7e3f-4b2a-9c1d-0a1b2c3d4e5f", + "implClass": "org.apache.ranger.services.RangerServiceDrill", + "version": 1, + "resources": [ + { + "itemId": 1, + "name": "datasource", + "label": "Data Source", + "description": "Drill storage plugin name (e.g. dfs, hive, mongo)", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "false" + }, + "validationRegEx": "", + "uiPageSize": "", + "accessTypeRestrictions": [] + }, + { + "itemId": 2, + "name": "schema", + "label": "Schema", + "description": "Schema path within the data source (e.g. tmp, default)", + "level": 20, + "parent": "datasource", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "validationRegEx": "", + "uiPageSize": "", + "accessTypeRestrictions": [] + }, + { + "itemId": 3, + "name": "table", + "label": "Table", + "description": "Table name within the schema", + "level": 30, + "parent": "schema", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "validationRegEx": "", + "uiPageSize": "", + "accessTypeRestrictions": [] + }, + { + "itemId": 4, + "name": "column", + "label": "Column", + "description": "Column name within the table", + "level": 40, + "parent": "table", + "mandatory": false, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "validationRegEx": "", + "uiPageSize": "", + "accessTypeRestrictions": [] + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "SELECT", + "label": "Select" + }, + { + "itemId": 2, + "name": "CREATE", + "label": "Create" + }, + { + "itemId": 3, + "name": "DROP", + "label": "Drop" + } + ], + "serviceConfigOptions": [ + { + "itemId": 1, + "name": "username", + "label": "Username", + "description": "Drill user name for policy lookup connection", + "type": "string", + "mandatory": true, + "defaultValue": "", + "validationRegEx": "" + }, + { + "itemId": 2, + "name": "password", + "label": "Password", + "description": "Drill user password", + "type": "password", + "mandatory": false, + "defaultValue": "" + }, + { + "itemId": 3, + "name": "drill.connection.url", + "label": "Drill REST URL", + "description": "Drill REST API endpoint for resource lookup (e.g. http://drillbit-host:8047)", + "type": "string", + "mandatory": true, + "defaultValue": "http://localhost:8047" + } + ], + "policyConditions": [], + "contextEnrichers": [], + "enums": [] +} diff --git a/distro/pom.xml b/distro/pom.xml index 1c4b224af0c..0918e7b54e7 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -114,6 +114,12 @@ ${project.version} provided + + org.apache.ranger + ranger-drill-plugin + ${project.version} + provided + org.apache.ranger ranger-elasticsearch-plugin diff --git a/distro/src/main/assembly/admin-web.xml b/distro/src/main/assembly/admin-web.xml index 40b36727c8d..d3250e2d2d7 100644 --- a/distro/src/main/assembly/admin-web.xml +++ b/distro/src/main/assembly/admin-web.xml @@ -486,6 +486,23 @@ + + true + + org.apache.ranger:ranger-drill-plugin + + + ews/webapp/WEB-INF/classes/ranger-plugins/drill + true + false + 755 + 644 + + org.apache.ranger:ranger-drill-plugin + + + + true diff --git a/plugin-drill/pom.xml b/plugin-drill/pom.xml new file mode 100644 index 00000000000..3706b6b47a7 --- /dev/null +++ b/plugin-drill/pom.xml @@ -0,0 +1,89 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger + 2.9.1-SNAPSHOT + .. + + + ranger-drill-plugin + jar + Ranger Security Plugin for Apache Drill + Ranger Security Plugin for Apache Drill (service-side resource lookup via Drill REST API) + + + UTF-8 + + + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + org.apache.ranger + ranger-common-utils + ${project.version} + + + org.apache.ranger + ranger-plugins-common + ${project.version} + + + javax.servlet + javax.servlet-api + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.junit.jupiter + junit-jupiter + ${junit.jupiter.version} + test + + + org.mockito + mockito-inline + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + diff --git a/plugin-drill/src/main/java/org/apache/ranger/services/RangerServiceDrill.java b/plugin-drill/src/main/java/org/apache/ranger/services/RangerServiceDrill.java new file mode 100644 index 00000000000..35f38f2acc7 --- /dev/null +++ b/plugin-drill/src/main/java/org/apache/ranger/services/RangerServiceDrill.java @@ -0,0 +1,565 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.services; + +import org.apache.commons.lang3.StringUtils; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.client.HadoopException; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.plugin.util.PasswordUtils; +import org.apache.ranger.plugin.util.TimedEventUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.ranger.plugin.client.BaseClient.generateResponseDataMap; + +/** + * Ranger service plugin for Apache Drill (service / Admin side). + * + *

Deployed into Ranger Admin. Provides: + *

    + *
  • {@link #validateConfig()} - tests connectivity to the Drill cluster by + * issuing a probe query through the Drill REST API.
  • + *
  • {@link #lookupResource(ResourceLookupContext)} - enumerates + * datasource / schema / table / column resources via + * {@code INFORMATION_SCHEMA} queries through the REST API, so the Ranger + * policy editor can auto-complete resource paths.
  • + *
+ * + *

Uses Drill's REST API (default port 8047) instead of JDBC so this module + * can be compiled with the same JDK as Ranger Admin and does not pull in + * Drill's JDK 21 dependencies. + * + *

Lookup semantics

+ *
    + *
  • userInput empty (null or "") -> list all under the upper-level constraints;
  • + *
  • userInput non-empty -> fuzzy match with {@code LIKE '%kw%'};
  • + *
  • any upper-level constraint missing (null or empty) -> return empty list + * (do not query); datasource is the only top-level resource;
  • + *
+ */ +public class RangerServiceDrill extends RangerBaseService { + + private static final Logger logger = LoggerFactory.getLogger(RangerServiceDrill.class); + + // Service config keys (must match ranger-servicedef-drill.json) + private static final String CONFIG_USERNAME = "username"; + private static final String CONFIG_PASSWORD = "password"; + private static final String CONFIG_DRILL_URL = "drill.connection.url"; + + // Resource names (must match ranger-servicedef-drill.json, lowercase per Ranger naming rules) + private static final String RESOURCE_DATASOURCE = "datasource"; + private static final String RESOURCE_SCHEMA = "schema"; + private static final String RESOURCE_TABLE = "table"; + private static final String RESOURCE_COLUMN = "column"; + + // HTTP connect / read timeout (milliseconds) + private static final int CONNECT_TIMEOUT_MS = 10_000; + private static final int READ_TIMEOUT_MS = 10_000; + private static final int LOOKUP_TIMEOUT_SECONDS = 5; + private static final int MAX_DATASOURCE = 50; + private static final int MAX_SCHEMA = 50; + private static final int MAX_TABLE_COLUMN = 100; + + // Access type used when injecting the read-only lookup user policy + private static final String ACCESS_TYPE_SELECT = "select"; + + // SQL templates. TABLES / COLUMNS are reserved keywords in Drill SQL and must be backtick-quoted. + // Runtime values are inserted via String.format("%s"); all user-supplied values are pre-escaped + // with escapeSql(...) to defuse single quotes. + private static final String SQL_VALIDATE = "SELECT 1"; + + // datasource: distinct first segment of SCHEMA_NAME. Two variants — list-all vs fuzzy match. + private static final String SQL_LOOKUP_DATASOURCE_ALL = + "SELECT DISTINCT SPLIT_PART(SCHEMA_NAME, '.', 1) AS DATASOURCE " + + "FROM INFORMATION_SCHEMA.SCHEMATA ORDER BY 1"; + private static final String SQL_LOOKUP_DATASOURCE_LIKE = + "SELECT DISTINCT SPLIT_PART(SCHEMA_NAME, '.', 1) AS DATASOURCE " + + "FROM INFORMATION_SCHEMA.SCHEMATA " + + "WHERE SPLIT_PART(SCHEMA_NAME, '.', 1) LIKE '%s' ORDER BY 1 '%s'"; + + // schema: list schemas under a datasource (LIKE '%' so single-segment sources like es still hit). + private static final String SQL_LOOKUP_SCHEMA = + "SELECT SCHEMA_NAME AS SCHEMA FROM INFORMATION_SCHEMA.SCHEMATA " + + "WHERE SCHEMA_NAME LIKE '%s' ORDER BY 1 LIMIT %s"; + + // table: list tables under a qualified (datasource[.schema]) name. + // args: %1$s = qualified name, %2$s = NOT IN(...) clause (may be empty), %3$s = LIKE/AND clause (may be empty). + private static final String SQL_LOOKUP_TABLE = + "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.`TABLES` " + + "WHERE TABLE_SCHEMA = '%1$s'%2$s%3$s ORDER BY 1 LIMIT %4$s"; + + // column: list columns under (qualified name, table). + // args: %1$s = qualified name, %2$s = table, %3$s = NOT IN(...) clause, %4$s = LIKE/AND clause. + private static final String SQL_LOOKUP_COLUMN = + "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.`COLUMNS` " + + "WHERE TABLE_SCHEMA = '%1$s' AND TABLE_NAME = '%2$s'%3$s%4$s ORDER BY 1 LIMIT %5$s"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + logger.debug("RangerServiceDrill initialized for service={}", + service != null ? service.getName() : "null"); + } + + /** + * Validates the service configuration by testing connectivity to the Drill + * cluster via a probe query through the Drill REST API. + * @return a map with {@code connectivityStatus=true} on success + * @throws HadoopException on any configuration or connectivity failure + */ + @Override + public Map validateConfig() throws Exception { + Map responseData = new HashMap<>(); + String responseMsg = ""; + String description = ""; + String response = ""; + String username = getConfig(CONFIG_USERNAME); + if (isBlank(username)) { + throw hadoopException("Drill user name is required", CONFIG_USERNAME); + } + String baseUrl = getConfig(CONFIG_DRILL_URL); + if (isBlank(baseUrl)) { + throw hadoopException("Drill connection URL is required", CONFIG_DRILL_URL); + } + String cryptPassword = getConfig(CONFIG_PASSWORD); + // Password may be stored encrypted (Ranger Admin auto-encrypts the "password" config key); + // decrypt it, and fall back to the raw value when it is already plaintext. + String decryptpassword = PasswordUtils.getDecryptPassword(cryptPassword); + + final String normalizedUrl; + try { + normalizedUrl = buildBaseUrl(baseUrl); + } catch (IllegalArgumentException e) { + throw hadoopException("Invalid drill.connection.url: " + e.getMessage(), CONFIG_DRILL_URL); + } + + logger.info("Validating Drill service connection to {}", normalizedUrl); + try { + response = queryWithTimeout(normalizedUrl, username, decryptpassword, SQL_VALIDATE); + JsonNode root = MAPPER.readTree(response); + if (root != null && root.has("rows") && root.get("rows").isArray()) { + responseMsg = "Drill connection test succeeded"; + description = "Connected to Drill at " + normalizedUrl; + generateResponseDataMap(true, responseMsg, description, + null, null, responseData); + logger.info("Drill connection validation succeeded for {}", normalizedUrl); + return responseData; + } + throw hadoopException("Unexpected response from Drill: " + truncate(response, 200), null); + } catch (Exception e) { + responseMsg = "Drill connection validation failed for url: " + normalizedUrl; + description = e.getMessage(); + generateResponseDataMap(false, responseMsg, description, + null, null, responseData); + } + return responseData; + } + + private static HadoopException hadoopException(String message, String fieldName) { + HadoopException he = new HadoopException(message); + he.generateResponseDataMap(false, message, message, null, fieldName); + return he; + } + + /** + * Lists Drill resources for the Ranger policy editor autocomplete. + * + *

Each resource level is constrained by the already-selected parent + * resources carried in {@link ResourceLookupContext#getResources()}. + * when {@code userInput} is empty,Drill lists everything under the constraints. + * @param context carries the requested resource name, the already-selected + * parents, and the optional user input + * @return a list of matching resource names. + */ + @Override + public List lookupResource(ResourceLookupContext context) throws Exception { + if (context == null) { + return Collections.emptyList(); + } + + String userInput = context.getUserInput() == null ? "" : context.getUserInput().trim(); + Map> resources = context.getResources() != null + ? context.getResources() : Collections.emptyMap(); + String resourceName = context.getResourceName(); + + if (isBlank(resourceName)) { + return Collections.emptyList(); + } + + List dsList = resources.get(RESOURCE_DATASOURCE); + List schemaList = resources.get(RESOURCE_SCHEMA); + List tableList = resources.get(RESOURCE_TABLE); + + try { + switch (resourceName) { + case RESOURCE_DATASOURCE: + // Top-level resource: no parent constraint required. + return lookupDatasource(userInput); + + case RESOURCE_SCHEMA: + // D8: missing datasource -> do not query, return empty. + if (isEmpty(dsList)) { + return Collections.emptyList(); + } + return lookupSchema(dsList, userInput); + + case RESOURCE_TABLE: + // D8: missing datasource or schema -> do not query. + if (isEmpty(dsList) || isEmpty(schemaList)) { + return Collections.emptyList(); + } + return lookupTable(dsList, schemaList, userInput, resources.get(RESOURCE_TABLE)); + + case RESOURCE_COLUMN: + if (isEmpty(dsList) || isEmpty(schemaList) || isEmpty(tableList)) { + return Collections.emptyList(); + } + return lookupColumn(dsList, schemaList, tableList, userInput, resources.get(RESOURCE_COLUMN)); + + default: + logger.warn("Unknown resource name: {}", resourceName); + return Collections.emptyList(); + } + } catch (Exception e) { + logger.error("lookupResource failed for resource={}, userInput={}", resourceName, userInput, e); + throw new RuntimeException("Resource lookup failed for " + resourceName + ": " + e.getMessage(), e); + } + } + + @Override + public List getDefaultRangerPolicies() throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerServiceDrill.getDefaultRangerPolicies()"); + } + List ret = super.getDefaultRangerPolicies(); + // inject a read-only "select" policy item for the lookup user + // into every default "all" policy, so the browse account can run lookups + // but cannot be made a delegate admin. + for (RangerPolicy defaultPolicy : ret) { + if (defaultPolicy.getName() != null + && defaultPolicy.getName().contains("all") + && StringUtils.isNotBlank(lookUpUser)) { + List accessList = new ArrayList<>(); + RangerPolicyItem policyItem = new RangerPolicyItem(); + accessList.add(new RangerPolicyItemAccess(ACCESS_TYPE_SELECT)); + policyItem.setUsers(Collections.singletonList(lookUpUser)); + policyItem.setAccesses(accessList); + policyItem.setDelegateAdmin(false); + + List policyItems = defaultPolicy.getPolicyItems(); + if (policyItems == null || policyItems.isEmpty()) { + policyItems = new ArrayList<>(); + } + policyItems.add(policyItem); + defaultPolicy.setPolicyItems(policyItems); + } + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerServiceDrill.getDefaultRangerPolicies()"); + } + return ret; + } + + /** datasource: distinct storage-plugin names = first segment of SCHEMA_NAME. */ + private List lookupDatasource(String userInput) throws Exception { + String sql; + if (userInput.isEmpty()) { + sql = SQL_LOOKUP_DATASOURCE_ALL; + } else { + sql = String.format(SQL_LOOKUP_DATASOURCE_LIKE, "%" + escapeSql(userInput) + "%", MAX_DATASOURCE); + } + return executeQuery(sql, "DATASOURCE"); + } + + /** + * schema: for each selected datasource, query {@code INFORMATION_SCHEMA.SCHEMATA} + * with {@code LIKE '%'} (so single-segment sources like {@code es} still hit), + * then split the SCHEMA_NAME by dots: + *

    + *
  • {@code a} (e.g. {@code es}) -> display {@code a};
  • + *
  • {@code a.b} (e.g. {@code dfs.tmp}) -> display {@code b} (second segment);
  • + *
  • {@code a.b.c} -> display {@code b} (second segment, no truncation);
  • + *
  • no match -> empty, never synthesised (fixes the earlier isEmpty-synthesise bug, plan P13).
  • + *
+ */ + private List lookupSchema(List dsList, String userInput) throws Exception { + List out = new ArrayList<>(); + String kw = userInput.toLowerCase(); + for (String ds : dsList) { + String sql = String.format(SQL_LOOKUP_SCHEMA, escapeSql(ds) + "%", MAX_SCHEMA); + List rows = executeQuery(sql, "SCHEMA"); + for (String schemaName : rows) { + String[] parts = schemaName.split("\\."); + String display = (parts.length == 1) ? parts[0] : parts[1]; + if (userInput.isEmpty() || display.toLowerCase().contains(kw)) { + out.add(display); + } + } + if (out.size() >= MAX_SCHEMA) { + break; + } + } + return out; + } + + /** table: for each (datasource, schema) pair, query under the qualified name, capped at {@link #MAX_TABLE_COLUMN}. */ + private List lookupTable(List dsList, List schemaList, + String userInput, List excluded) throws Exception { + List out = new ArrayList<>(); + String notIn = (excluded != null && !excluded.isEmpty()) + ? " AND TABLE_NAME NOT IN (" + sqlIn(excluded) + ")" : ""; + String like = userInput.isEmpty() ? "" : " AND TABLE_NAME LIKE '%" + escapeSql(userInput) + "%'"; + for (String ds : dsList) { + for (String schema : schemaList) { + String qualified = ds.equals(schema) ? ds : ds + "." + schema; // es: es == es -> "es" + String sql = String.format(SQL_LOOKUP_TABLE, escapeSql(qualified), notIn, like, MAX_TABLE_COLUMN); + for (String t : executeQuery(sql, "TABLE_NAME")) { + if (out.size() >= MAX_TABLE_COLUMN) { + return out; + } + out.add(t); + } + } + } + return out; + } + + /** column: for each selected table, query under (qualified name, table), cross-table total capped at {@link #MAX_TABLE_COLUMN}. */ + private List lookupColumn(List dsList, List schemaList, List tableList, + String userInput, List excluded) throws Exception { + List out = new ArrayList<>(); + String notIn = (excluded != null && !excluded.isEmpty()) + ? " AND COLUMN_NAME NOT IN (" + sqlIn(excluded) + ")" : ""; + String like = userInput.isEmpty() ? "" : " AND COLUMN_NAME LIKE '%" + escapeSql(userInput) + "%'"; + for (String ds : dsList) { + for (String schema : schemaList) { + String qualified = ds.equals(schema) ? ds : ds + "." + schema; + for (String tbl : tableList) { + String sql = String.format(SQL_LOOKUP_COLUMN, + escapeSql(qualified), escapeSql(tbl), notIn, like, MAX_TABLE_COLUMN); + for (String c : executeQuery(sql, "COLUMN_NAME")) { + if (out.size() >= MAX_TABLE_COLUMN) { + return out; + } + out.add(c); + } + } + } + } + return out; + } + + // ======================================================================== + // REST API helpers + // ======================================================================== + + /** + * Executes a SQL query via the Drill REST API and returns the first-column + * values. The whole round-trip is bounded by {@link #LOOKUP_TIMEOUT_SECONDS} + * using {@link TimedEventUtil}. + */ + private List executeQuery(String sql, String columnAlias) throws Exception { + String baseUrl = buildBaseUrl(getConfig(CONFIG_DRILL_URL)); + String json = queryWithTimeout(baseUrl, getConfig(CONFIG_USERNAME), + PasswordUtils.getDecryptPassword(getConfig(CONFIG_PASSWORD)), sql); + return extractFirstColumnValues(json, columnAlias); + } + + private String queryWithTimeout(String baseUrl, String username, String password, String sql) throws Exception { + return TimedEventUtil.timedTask( + () -> executeRaw(baseUrl, username, password, sql), + LOOKUP_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + /** + * Executes a SQL query via the Drill REST API. + *

POSTs to {@code /query.json} with a JSON body + * {@code {"queryType":"SQL","query":""}} using HTTP Basic auth.

+ */ + private static String executeRaw(String baseUrl, String username, String password, String sql) + throws Exception { + String endpoint = baseUrl + "/query.json"; + Map payload = new HashMap<>(); + payload.put("queryType", "SQL"); + payload.put("query", sql); + String body = MAPPER.writeValueAsString(payload); + + HttpURLConnection conn = null; + try { + URL url = new URL(endpoint); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setConnectTimeout(CONNECT_TIMEOUT_MS); + conn.setReadTimeout(READ_TIMEOUT_MS); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("Accept", "application/json"); + + // HTTP Basic auth + String creds = username + ":" + (password == null ? "" : password); + String encoded = Base64.getEncoder().encodeToString(creds.getBytes(StandardCharsets.UTF_8)); + conn.setRequestProperty("Authorization", "Basic " + encoded); + + try (OutputStream os = conn.getOutputStream()) { + os.write(body.getBytes(StandardCharsets.UTF_8)); + } + + int code = conn.getResponseCode(); + InputStream is = (code >= 200 && code < 300) ? conn.getInputStream() : conn.getErrorStream(); + String response = readAll(is); + + if (code < 200 || code >= 300) { + throw new RuntimeException("Drill REST API returned HTTP " + code + ": " + truncate(response, 500)); + } + return response; + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + /** + * Extracts the values of the named column from a Drill REST API response. + *

Drill returns JSON like: + *

{@code {"columns":["COL1"],"rows":[{"COL1":"value1"},{"COL1":"value2"}]}}
+ */ + private static List extractFirstColumnValues(String jsonResponse, String columnAlias) throws Exception { + JsonNode root = MAPPER.readTree(jsonResponse); + String queryState = root.get("queryState").asText(); + if (root == null || !root.has("rows") || !root.get("rows").isArray() || "FAILED".equals(queryState)) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + for (JsonNode row : root.get("rows")) { + if (row.has(columnAlias)) { + String value = row.get(columnAlias).asText(); + if (value != null && !value.trim().isEmpty()) { + result.add(value.trim()); + } + } else if (row.isObject() && row.size() > 0) { + // Fallback: first field of the row + String value = row.elements().next().asText(); + if (value != null && !value.trim().isEmpty()) { + result.add(value.trim()); + } + } + } + return result; + } + + static String buildBaseUrl(String configuredUrl) { + if (isBlank(configuredUrl)) { + throw new IllegalArgumentException("drill.connection.url is empty"); + } + String url = configuredUrl.trim(); + while (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + return url; + } + + private String getConfig(String key) { + Map cfg = getConfigs(); + if (cfg == null) { + return null; + } + Object value = cfg.get(key); + return value == null ? null : value.toString().trim(); + } + + private static String escapeSql(String value) { + // Only single-quote escaping is applied; '%' / '_' are intentionally + // allowed as LIKE wildcards. For stricter behaviour add an + // ESCAPE clause and escape those characters here. + return value == null ? "" : value.replace("'", "''"); + } + + private static String sqlIn(List values) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < values.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append("'").append(escapeSql(values.get(i))).append("'"); + } + return sb.toString(); + } + + private static List cap(List rows, int limit) { + if (rows.size() <= limit) { + return rows; + } + return new ArrayList<>(rows.subList(0, limit)); + } + + private static boolean isEmpty(List list) { + return list == null || list.isEmpty(); + } + + private static boolean isBlank(String s) { + return s == null || s.trim().isEmpty(); + } + + private static String truncate(String s, int maxLen) { + if (s == null) { + return ""; + } + return s.length() <= maxLen ? s : s.substring(0, maxLen) + "..."; + } + + private static String readAll(InputStream is) throws Exception { + if (is == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + sb.append(line).append('\n'); + } + } + return sb.toString(); + } +} diff --git a/plugin-drill/src/test/java/org/apache/ranger/services/RangerServiceDrillTest.java b/plugin-drill/src/test/java/org/apache/ranger/services/RangerServiceDrillTest.java new file mode 100644 index 00000000000..289885e0a33 --- /dev/null +++ b/plugin-drill/src/test/java/org/apache/ranger/services/RangerServiceDrillTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import org.apache.ranger.plugin.client.HadoopException; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.plugin.util.PasswordUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link RangerServiceDrill}. + * + *

Uses JDK's built-in {@link HttpServer} to stand in for the Drill REST + * endpoint ({@code POST /query.json}). The tests assert that + *

    + *
  • the password stored by Ranger Admin (encrypted 5-segment value) is + * decrypted before it is used for HTTP Basic auth, and
  • + *
  • a plain-text password falls back to the raw value instead of failing + * the connection test ({@code PasswordUtils.getDecryptPassword}), and
  • + *
  • expected lookup behaviours (parent-gated resources, empty result + * handling) hold while talking to a real HTTP stack.
  • + *
+ */ +public class RangerServiceDrillTest { + + private static final String CONFIG_DRILL_URL = "drill.connection.url"; + private static final String CONFIG_USERNAME = "username"; + private static final String CONFIG_PASSWORD = "password"; + + private static final String TEST_USERNAME = "rangerlookup"; + private static final String TEST_PLAIN_PASSWORD = "s3cretDri11!"; + + private HttpServer server; + private volatile String capturedAuth; + private volatile String capturedBody; + private volatile int responseCode = 200; + private volatile String responseBody = "{\"rows\":[]}"; + + @BeforeEach + void startFakeDrillServer() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", this::handle); + server.start(); + } + + @AfterEach + void stopFakeDrillServer() { + if (server != null) { + server.stop(0); + } + } + + private void handle(HttpExchange exchange) throws IOException { + capturedAuth = exchange.getRequestHeaders().getFirst("Authorization"); + capturedBody = readRequestBody(exchange.getRequestBody()); + byte[] resp = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(responseCode, resp.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(resp); + } + } + + /** Reads the whole request body. JDK 8 compatible replacement for {@code InputStream.readAllBytes()} (Java 9+). */ + private static String readRequestBody(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int n; + while ((n = in.read(buf)) != -1) { + out.write(buf, 0, n); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + + private String baseUrl() { + return "http://127.0.0.1:" + server.getAddress().getPort(); + } + + private RangerServiceDrill newService(String username, String password) { + Map configs = new HashMap<>(); + configs.put(CONFIG_DRILL_URL, baseUrl()); + configs.put(CONFIG_USERNAME, username); + configs.put(CONFIG_PASSWORD, password); + RangerServiceDrill service = new RangerServiceDrill(); + service.setConfigs(configs); + return service; + } + + /** + * Builds the 5-segment encrypted password value exactly like Ranger Admin's + * {@code ServiceDBStore} does for the default PBEWithMD5AndDES algorithm: + * {@code algo,key,salt,iterations,encryptedPayload}. The payload itself is + * the plain text encrypted with the very same default parameters. + */ + private static String rangerStoredPassword(String plainPassword) throws Exception { + String head = PasswordUtils.DEFAULT_CRYPT_ALGO + "," + PasswordUtils.DEFAULT_ENCRYPT_KEY + "," + + PasswordUtils.DEFAULT_SALT + "," + PasswordUtils.DEFAULT_ITERATION_COUNT; + return head + "," + PasswordUtils.encryptPassword(head + "," + plainPassword); + } + + private static String expectedBasicAuth(String username, String password) { + String creds = username + ":" + password; + return "Basic " + Base64.getEncoder().encodeToString(creds.getBytes(StandardCharsets.UTF_8)); + } + + // --------------------------------------------------------------------- + // validateConfig + // --------------------------------------------------------------------- + + @Test + void validateConfig_withEncryptedPassword_decryptsBeforeBasicAuth() throws Exception { + String stored = rangerStoredPassword(TEST_PLAIN_PASSWORD); + + Map result = newService(TEST_USERNAME, stored).validateConfig(); + + assertEquals(Boolean.TRUE, result.get("connectivityStatus")); + assertEquals(expectedBasicAuth(TEST_USERNAME, TEST_PLAIN_PASSWORD), capturedAuth, + "Basic auth must use the decrypted plain-text password"); + } + + @Test + void validateConfig_withPlainTextPassword_fallsBackToRawValue() throws Exception { + Map result = newService(TEST_USERNAME, TEST_PLAIN_PASSWORD).validateConfig(); + + assertEquals(Boolean.TRUE, result.get("connectivityStatus")); + assertEquals(expectedBasicAuth(TEST_USERNAME, TEST_PLAIN_PASSWORD), capturedAuth, + "Plain-text password must be used as-is (no decrypt failure)"); + } + + @Test + void validateConfig_withoutUsername_throwsHadoopExceptionWithFieldName() { + assertThrows(HadoopException.class, + () -> newService("", TEST_PLAIN_PASSWORD).validateConfig()); + } + + @Test + void validateConfig_withNonJsonBody_returnsConnectivityFalse() throws Exception { + responseBody = "{\"foo\":\"bar\"}"; + + Map result = newService(TEST_USERNAME, TEST_PLAIN_PASSWORD).validateConfig(); + + assertEquals(Boolean.FALSE, result.get("connectivityStatus")); + } + + @Test + void validateConfig_withHttpError_returnsConnectivityFalse() throws Exception { + responseCode = 500; + responseBody = "{\"error\":\"boom\"}"; + + Map result = newService(TEST_USERNAME, TEST_PLAIN_PASSWORD).validateConfig(); + + assertEquals(Boolean.FALSE, result.get("connectivityStatus")); + } + + // --------------------------------------------------------------------- + // lookupResource + // --------------------------------------------------------------------- + + @Test + void lookupResource_datasource_withEncryptedPassword_returnsRowsAndDecryptedAuth() throws Exception { + responseBody = "{\"queryState\":\"COMPLETED\",\"columns\":[\"DATASOURCE\"]," + + "\"rows\":[{\"DATASOURCE\":\"dfs\"},{\"DATASOURCE\":\"dfs.default\"}]}"; + ResourceLookupContext ctx = new ResourceLookupContext(); + ctx.setResourceName("datasource"); + ctx.setUserInput(""); + + List result = newService(TEST_USERNAME, rangerStoredPassword(TEST_PLAIN_PASSWORD)) + .lookupResource(ctx); + + assertEquals(Arrays.asList("dfs", "dfs.default"), result); + assertEquals(expectedBasicAuth(TEST_USERNAME, TEST_PLAIN_PASSWORD), capturedAuth, + "lookup must decrypt the stored password before Basic auth"); + } + + @Test + void lookupResource_schema_withoutParentDatasource_returnsEmptyWithoutQuery() throws Exception { + ResourceLookupContext ctx = new ResourceLookupContext(); + ctx.setResourceName("schema"); + ctx.setUserInput(""); + ctx.setResources(new HashMap<>()); + + List result = newService(TEST_USERNAME, TEST_PLAIN_PASSWORD).lookupResource(ctx); + + assertEquals(Collections.emptyList(), result); + assertNull(capturedBody, "No HTTP call must be made when the parent datasource is missing"); + } + + @Test + void lookupResource_withBlankResourceName_returnsEmpty() throws Exception { + ResourceLookupContext ctx = new ResourceLookupContext(); + ctx.setResourceName(" "); + + List result = newService(TEST_USERNAME, TEST_PLAIN_PASSWORD).lookupResource(ctx); + + assertEquals(Collections.emptyList(), result); + assertNull(capturedBody); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index e0426a5015d..349a8069925 100755 --- a/pom.xml +++ b/pom.xml @@ -803,6 +803,7 @@ knox-agent pdp plugin-atlas + plugin-drill plugin-elasticsearch plugin-kafka plugin-kms @@ -1103,11 +1104,25 @@ agents-installer common-utils credentialbuilder + plugin-drill plugin-trino ranger-plugin-classloader ranger-util + + ranger-drill-plugin + + agents-audit + agents-common + agents-cred + agents-installer + common-utils + plugin-drill + ranger-plugin-classloader + ranger-util + + ranger-nestedstructure-plugin @@ -1153,6 +1168,7 @@ knox-agent pdp plugin-atlas + plugin-drill plugin-elasticsearch plugin-kafka plugin-kms @@ -1242,6 +1258,7 @@ knox-agent pdp plugin-atlas + plugin-drill plugin-elasticsearch plugin-kafka plugin-kms