You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2025/01/13 10:12:42 UTC
(ranger) 04/14: RANGER-5061: security-admin module(elasticsearch package): update for code readability improvement (#488)
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch RANGER-5061_master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 9c3ac972f60bba28a6751d830b51d1ef238d484f
Author: Vyom Mani Tiwari <vy...@gmail.com>
AuthorDate: Sat Jan 4 00:47:39 2025 +0530
RANGER-5061: security-admin module(elasticsearch package): update for code readability improvement (#488)
* RANGER-5069: Add ability to Kafka authorizer to define super users through Kafka config
Co-authored-by: Daniel Fonai <df...@cloudera.com>
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../ElasticSearchAccessAuditsService.java | 541 +++++++++++----------
.../ranger/elasticsearch/ElasticSearchMgr.java | 220 +++++----
.../ranger/elasticsearch/ElasticSearchUtil.java | 130 +++--
3 files changed, 486 insertions(+), 405 deletions(-)
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
index a64cb111e..4f6c808ff 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
@@ -27,10 +27,10 @@ import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.db.XXServiceDefDao;
import org.apache.ranger.entity.XXService;
import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.plugin.util.JsonUtilsV2;
import org.apache.ranger.view.VXAccessAudit;
import org.apache.ranger.view.VXAccessAuditList;
import org.apache.ranger.view.VXLong;
-import org.apache.ranger.plugin.util.JsonUtilsV2;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.search.SearchResponse;
@@ -50,260 +50,289 @@ import java.util.Map;
@Service
@Scope("singleton")
public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAuditsService {
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchAccessAuditsService.class);
-
- @Autowired
- ElasticSearchMgr elasticSearchMgr;
-
- @Autowired
- ElasticSearchUtil elasticSearchUtil;
-
-
- public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
-
- RestHighLevelClient client = elasticSearchMgr.getClient();
- final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true);
- if (client == null) {
- LOGGER.warn("ElasticSearch client is null, so not running the query.");
- throw restErrorUtil.createRESTException(
- "Error connecting to search engine",
- MessageEnums.ERROR_SYSTEM);
- }
- List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
- Map<String, Object> paramList = searchCriteria.getParamList();
- updateUserExclusion(paramList);
-
- SearchResponse response;
- try {
- response = elasticSearchUtil.searchResources(searchCriteria, searchFields, sortFields, client, elasticSearchMgr.index);
- } catch (IOException e) {
- LOGGER.warn(String.format("ElasticSearch query failed: %s", e.getMessage()));
- throw restErrorUtil.createRESTException(
- "Error querying search engine",
- MessageEnums.ERROR_SYSTEM);
- }
- MultiGetItemResponse[] docs;
- try {
- docs = elasticSearchUtil.fetch(client, elasticSearchMgr.index, response.getHits().getHits());
- } catch (IOException e) {
- LOGGER.warn(String.format("ElasticSearch fetch failed: %s", e.getMessage()));
- throw restErrorUtil.createRESTException(
- "Error querying search engine",
- MessageEnums.ERROR_SYSTEM);
- }
- for (int i = 0; i < docs.length; i++) { // NOPMD - This for loop can be replaced by a foreach loop
- MultiGetItemResponse doc = docs[i];
- VXAccessAudit vXAccessAudit = populateViewBean(doc.getResponse());
- if (vXAccessAudit != null) {
- String serviceType = vXAccessAudit.getServiceType();
- boolean isHive = "hive".equalsIgnoreCase(serviceType);
- if (!hiveQueryVisibility && isHive) {
- vXAccessAudit.setRequestData(null);
- } else if (isHive) {
- String accessType = vXAccessAudit.getAccessType();
- if ("grant".equalsIgnoreCase(accessType)
- || "revoke".equalsIgnoreCase(accessType)) {
- String requestData = vXAccessAudit.getRequestData();
- if (requestData != null) {
- try {
- vXAccessAudit.setRequestData(
- java.net.URLDecoder.decode(requestData, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- LOGGER.warn("Error while encoding request data: " + requestData, e);
- }
- } else {
- LOGGER.warn(
- "Error in request data of audit from elasticSearch. AuditData: "
- + vXAccessAudit.toString());
- }
- }
- }
- }
- xAccessAuditList.add(vXAccessAudit);
- }
-
- VXAccessAuditList returnList = new VXAccessAuditList();
- returnList.setPageSize(searchCriteria.getMaxRows());
- returnList.setResultSize(response.getHits().getHits().length);
- returnList.setTotalCount(response.getHits().getTotalHits().value);
- returnList.setStartIndex(searchCriteria.getStartIndex());
- returnList.setVXAccessAudits(xAccessAuditList);
- return returnList;
- }
-
- public void setRestErrorUtil(RESTErrorUtil restErrorUtil) {
- this.restErrorUtil = restErrorUtil;
- }
-
-
- /**
- * @param doc
- * @return
- */
- private VXAccessAudit populateViewBean(GetResponse doc) {
- VXAccessAudit accessAudit = new VXAccessAudit();
-
- Object value = null;
- if(LOGGER.isDebugEnabled()) {
- LOGGER.debug("doc=" + doc.toString());
- }
-
- Map<String, Object> source = doc.getSource();
- value = source.get("id");
- if (value != null) {
- // TODO: Converting ID to hashcode for now
- accessAudit.setId((long) value.hashCode());
- }
-
- value = source.get("cluster");
- if (value != null) {
- accessAudit.setClusterName(value.toString());
- }
-
- value = source.get("zoneName");
- if (value != null) {
- accessAudit.setZoneName(value.toString());
- }
-
- value = source.get("agentHost");
- if (value != null) {
- accessAudit.setAgentHost(value.toString());
- }
-
- value = source.get("policyVersion");
- if (value != null) {
- accessAudit.setPolicyVersion(MiscUtil.toLong(value));
- }
-
- value = source.get("access");
- if (value != null) {
- accessAudit.setAccessType(value.toString());
- }
-
- value = source.get("enforcer");
- if (value != null) {
- accessAudit.setAclEnforcer(value.toString());
- }
- value = source.get("agent");
- if (value != null) {
- accessAudit.setAgentId(value.toString());
- }
- value = source.get("repo");
- if (value != null) {
- accessAudit.setRepoName(value.toString());
- XXService xxService = daoManager.getXXService().findByName(accessAudit.getRepoName());
-
- if(xxService != null) {
- accessAudit.setRepoDisplayName(xxService.getDisplayName());
- }
- }
- value = source.get("sess");
- if (value != null) {
- accessAudit.setSessionId(value.toString());
- }
- value = source.get("reqUser");
- if (value != null) {
- accessAudit.setRequestUser(value.toString());
- }
- value = source.get("reqData");
- if (value != null) {
- accessAudit.setRequestData(value.toString());
- }
- value = source.get("resource");
- if (value != null) {
- accessAudit.setResourcePath(value.toString());
- }
- value = source.get("cliIP");
- if (value != null) {
- accessAudit.setClientIP(value.toString());
- }
- value = source.get("logType");
- //if (value != null) {
- // TODO: Need to see what logType maps to in UI
-// accessAudit.setAuditType(solrUtil.toInt(value));
- //}
- value = source.get("result");
- if (value != null) {
- accessAudit.setAccessResult(MiscUtil.toInt(value));
- }
- value = source.get("policy");
- if (value != null) {
- accessAudit.setPolicyId(MiscUtil.toLong(value));
- }
- value = source.get("repoType");
- if (value != null) {
- accessAudit.setRepoType(MiscUtil.toInt(value));
- if(null != daoManager) {
- XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef();
- if(xxServiceDef != null) {
- XXServiceDef xServiceDef = xxServiceDef.getById((long) accessAudit.getRepoType());
- if (xServiceDef != null) {
- accessAudit.setServiceType(xServiceDef.getName());
- accessAudit.setServiceTypeDisplayName(xServiceDef.getDisplayName());
- }
- }
- }
- }
- value = source.get("resType");
- if (value != null) {
- accessAudit.setResourceType(value.toString());
- }
- value = source.get("reason");
- if (value != null) {
- accessAudit.setResultReason(value.toString());
- }
- value = source.get("action");
- if (value != null) {
- accessAudit.setAction(value.toString());
- }
- value = source.get("evtTime");
- if (value != null) {
- accessAudit.setEventTime(MiscUtil.toLocalDate(value));
- }
- value = source.get("seq_num");
- if (value != null) {
- accessAudit.setSequenceNumber(MiscUtil.toLong(value));
- }
- value = source.get("event_count");
- if (value != null) {
- accessAudit.setEventCount(MiscUtil.toLong(value));
- }
- value = source.get("event_dur_ms");
- if (value != null) {
- accessAudit.setEventDuration(MiscUtil.toLong(value));
- }
- value = source.get("tags");
- if (value != null) {
- accessAudit.setTags(value.toString());
- }
- value = source.get("datasets");
- if (value != null) {
- try {
- accessAudit.setDatasets(JsonUtilsV2.nonSerializableObjToJson(value));
- } catch (Exception e) {
- LOGGER.warn("Failed to convert datasets to json", e);
- }
- }
- value = source.get("projects");
- if (value != null) {
- try {
- accessAudit.setProjects(JsonUtilsV2.nonSerializableObjToJson(value));
- } catch (Exception e) {
- LOGGER.warn("Failed to convert projects to json", e);
- }
- }
- return accessAudit;
- }
-
- /**
- * @param searchCriteria
- * @return
- */
- public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
- long count = 100;
- VXLong vXLong = new VXLong();
- vXLong.setValue(count);
- return vXLong;
- }
+ private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchAccessAuditsService.class);
+
+ @Autowired
+ ElasticSearchMgr elasticSearchMgr;
+
+ @Autowired
+ ElasticSearchUtil elasticSearchUtil;
+
+ public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
+ RestHighLevelClient client = elasticSearchMgr.getClient();
+ final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true);
+
+ if (client == null) {
+ LOGGER.warn("ElasticSearch client is null, so not running the query.");
+
+ throw restErrorUtil.createRESTException("Error connecting to search engine", MessageEnums.ERROR_SYSTEM);
+ }
+
+ List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
+ Map<String, Object> paramList = searchCriteria.getParamList();
+
+ updateUserExclusion(paramList);
+
+ SearchResponse response;
+
+ try {
+ response = elasticSearchUtil.searchResources(searchCriteria, searchFields, sortFields, client, elasticSearchMgr.index);
+ } catch (IOException e) {
+ LOGGER.warn("ElasticSearch query failed: {}", e.getMessage());
+
+ throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM);
+ }
+
+ MultiGetItemResponse[] docs;
+
+ try {
+ docs = elasticSearchUtil.fetch(client, elasticSearchMgr.index, response.getHits().getHits());
+ } catch (IOException e) {
+ LOGGER.warn("ElasticSearch fetch failed: {}", e.getMessage());
+
+ throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM);
+ }
+
+ for (int i = 0; i < docs.length; i++) { // NOPMD - This for loop can be replaced by a foreach loop
+ MultiGetItemResponse doc = docs[i];
+ VXAccessAudit vXAccessAudit = populateViewBean(doc.getResponse());
+
+ if (vXAccessAudit != null) {
+ String serviceType = vXAccessAudit.getServiceType();
+ boolean isHive = "hive".equalsIgnoreCase(serviceType);
+
+ if (!hiveQueryVisibility && isHive) {
+ vXAccessAudit.setRequestData(null);
+ } else if (isHive) {
+ String accessType = vXAccessAudit.getAccessType();
+
+ if ("grant".equalsIgnoreCase(accessType) || "revoke".equalsIgnoreCase(accessType)) {
+ String requestData = vXAccessAudit.getRequestData();
+
+ if (requestData != null) {
+ try {
+ vXAccessAudit.setRequestData(java.net.URLDecoder.decode(requestData, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.warn("Error while encoding request data: {}", requestData, e);
+ }
+ } else {
+ LOGGER.warn("Error in request data of audit from elasticSearch. AuditData: {}", vXAccessAudit);
+ }
+ }
+ }
+ }
+
+ xAccessAuditList.add(vXAccessAudit);
+ }
+
+ VXAccessAuditList returnList = new VXAccessAuditList();
+
+ returnList.setPageSize(searchCriteria.getMaxRows());
+ returnList.setResultSize(response.getHits().getHits().length);
+ returnList.setTotalCount(response.getHits().getTotalHits().value);
+ returnList.setStartIndex(searchCriteria.getStartIndex());
+ returnList.setVXAccessAudits(xAccessAuditList);
+
+ return returnList;
+ }
+
+ public void setRestErrorUtil(RESTErrorUtil restErrorUtil) {
+ this.restErrorUtil = restErrorUtil;
+ }
+
+ /**
+ * @param searchCriteria
+ * @return
+ */
+ public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
+ long count = 100;
+ VXLong vXLong = new VXLong();
+
+ vXLong.setValue(count);
+
+ return vXLong;
+ }
+
+ /**
+ * @param doc
+ * @return
+ */
+ private VXAccessAudit populateViewBean(GetResponse doc) {
+ LOGGER.debug("doc={}", doc);
+
+ VXAccessAudit accessAudit = new VXAccessAudit();
+ Map<String, Object> source = doc.getSource();
+ Object value;
+
+ value = source.get("id");
+ if (value != null) {
+ // TODO: Converting ID to hashcode for now
+ accessAudit.setId((long) value.hashCode());
+ }
+
+ value = source.get("cluster");
+ if (value != null) {
+ accessAudit.setClusterName(value.toString());
+ }
+
+ value = source.get("zoneName");
+ if (value != null) {
+ accessAudit.setZoneName(value.toString());
+ }
+
+ value = source.get("agentHost");
+ if (value != null) {
+ accessAudit.setAgentHost(value.toString());
+ }
+
+ value = source.get("policyVersion");
+ if (value != null) {
+ accessAudit.setPolicyVersion(MiscUtil.toLong(value));
+ }
+
+ value = source.get("access");
+ if (value != null) {
+ accessAudit.setAccessType(value.toString());
+ }
+
+ value = source.get("enforcer");
+ if (value != null) {
+ accessAudit.setAclEnforcer(value.toString());
+ }
+
+ value = source.get("agent");
+ if (value != null) {
+ accessAudit.setAgentId(value.toString());
+ }
+
+ value = source.get("repo");
+ if (value != null) {
+ accessAudit.setRepoName(value.toString());
+
+ XXService xxService = daoManager.getXXService().findByName(accessAudit.getRepoName());
+
+ if (xxService != null) {
+ accessAudit.setRepoDisplayName(xxService.getDisplayName());
+ }
+ }
+
+ value = source.get("sess");
+ if (value != null) {
+ accessAudit.setSessionId(value.toString());
+ }
+
+ value = source.get("reqUser");
+ if (value != null) {
+ accessAudit.setRequestUser(value.toString());
+ }
+
+ value = source.get("reqData");
+ if (value != null) {
+ accessAudit.setRequestData(value.toString());
+ }
+
+ value = source.get("resource");
+ if (value != null) {
+ accessAudit.setResourcePath(value.toString());
+ }
+
+ value = source.get("cliIP");
+ if (value != null) {
+ accessAudit.setClientIP(value.toString());
+ }
+
+ // TODO: Need to see what logType maps to in UI
+ //value = source.get("logType");
+ //if (value != null) {
+ // accessAudit.setAuditType(solrUtil.toInt(value));
+ //}
+
+ value = source.get("result");
+ if (value != null) {
+ accessAudit.setAccessResult(MiscUtil.toInt(value));
+ }
+
+ value = source.get("policy");
+ if (value != null) {
+ accessAudit.setPolicyId(MiscUtil.toLong(value));
+ }
+
+ value = source.get("repoType");
+ if (value != null) {
+ accessAudit.setRepoType(MiscUtil.toInt(value));
+
+ if (null != daoManager) {
+ XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef();
+
+ if (xxServiceDef != null) {
+ XXServiceDef xServiceDef = xxServiceDef.getById((long) accessAudit.getRepoType());
+
+ if (xServiceDef != null) {
+ accessAudit.setServiceType(xServiceDef.getName());
+ accessAudit.setServiceTypeDisplayName(xServiceDef.getDisplayName());
+ }
+ }
+ }
+ }
+
+ value = source.get("resType");
+ if (value != null) {
+ accessAudit.setResourceType(value.toString());
+ }
+
+ value = source.get("reason");
+ if (value != null) {
+ accessAudit.setResultReason(value.toString());
+ }
+
+ value = source.get("action");
+ if (value != null) {
+ accessAudit.setAction(value.toString());
+ }
+
+ value = source.get("evtTime");
+ if (value != null) {
+ accessAudit.setEventTime(MiscUtil.toLocalDate(value));
+ }
+
+ value = source.get("seq_num");
+ if (value != null) {
+ accessAudit.setSequenceNumber(MiscUtil.toLong(value));
+ }
+
+ value = source.get("event_count");
+ if (value != null) {
+ accessAudit.setEventCount(MiscUtil.toLong(value));
+ }
+
+ value = source.get("event_dur_ms");
+ if (value != null) {
+ accessAudit.setEventDuration(MiscUtil.toLong(value));
+ }
+
+ value = source.get("tags");
+ if (value != null) {
+ accessAudit.setTags(value.toString());
+ }
+
+ value = source.get("datasets");
+ if (value != null) {
+ try {
+ accessAudit.setDatasets(JsonUtilsV2.nonSerializableObjToJson(value));
+ } catch (Exception e) {
+ LOGGER.warn("Failed to convert datasets to json", e);
+ }
+ }
+
+ value = source.get("projects");
+ if (value != null) {
+ try {
+ accessAudit.setProjects(JsonUtilsV2.nonSerializableObjToJson(value));
+ } catch (Exception e) {
+ LOGGER.warn("Failed to convert projects to json", e);
+ }
+ }
+ return accessAudit;
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java
index 6987f6eb1..289578290 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java
@@ -47,110 +47,134 @@ import java.security.PrivilegedActionException;
import java.util.Date;
import java.util.Locale;
-import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.*;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_INDEX;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PORT;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PREFIX;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PROTOCOL;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PWRD;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_URLS;
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_USER;
/**
* This class initializes the ElasticSearch client
- *
*/
@Component
public class ElasticSearchMgr {
+ private static final Logger logger = LoggerFactory.getLogger(ElasticSearchMgr.class);
+
+ public String index;
+
+ Subject subject;
+ String user;
+ String password;
+ RestHighLevelClient client;
+
+ public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) {
+ RestClientBuilder restClientBuilder = RestClient.builder(MiscUtil.toArray(urls, ",").stream().map(x -> new HttpHost(x, port, protocol)).<HttpHost>toArray(i -> new HttpHost[i]));
+
+ if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) {
+ if (password.contains("keytab") && new File(password).exists()) {
+ final KerberosCredentialsProvider credentialsProvider = CredentialsProviderUtil.getKerberosCredentials(user, password);
+ Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
+
+ restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
+ clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+ clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+
+ return clientBuilder;
+ });
+ } else {
+ final CredentialsProvider credentialsProvider = CredentialsProviderUtil.getBasicCredentials(user, password);
+
+ restClientBuilder.setHttpClientConfigCallback(clientBuilder -> clientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+ }
+ } else {
+ logger.error("ElasticSearch Credentials not provided!!");
+
+ final CredentialsProvider credentialsProvider = null;
+
+ restClientBuilder.setHttpClientConfigCallback(clientBuilder -> clientBuilder.setDefaultCredentialsProvider(credentialsProvider));
+ }
+
+ return restClientBuilder;
+ }
+
+ public RestHighLevelClient getClient() {
+ RestHighLevelClient me = client;
+
+ if (me != null && subject != null) {
+ KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject);
+
+ try {
+ if (new Date().getTime() > ticket.getEndTime().getTime()) {
+ client = null;
+ CredentialsProviderUtil.ticketExpireTime80 = 0;
+
+ me = connect();
+ } else if (CredentialsProviderUtil.ticketWillExpire(ticket)) {
+ subject = CredentialsProviderUtil.login(user, password);
+ }
+ } catch (PrivilegedActionException e) {
+ logger.error("PrivilegedActionException:", e);
+
+ throw new RuntimeException(e);
+ }
+
+ return me;
+ } else {
+ me = connect();
+ }
+
+ return me;
+ }
+
+ synchronized RestHighLevelClient connect() {
+ RestHighLevelClient me = client;
+
+ if (me == null) {
+ synchronized (ElasticSearchAuditDestination.class) {
+ me = client;
+
+ if (me == null) {
+ String urls = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_URLS);
+ String protocol = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PROTOCOL, "http");
+
+ user = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_USER, "");
+ password = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PWRD, "");
+
+ int port = Integer.parseInt(PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PORT));
+
+ this.index = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_INDEX, "ranger_audits");
+
+ String parameterString = String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, urls, port, index);
+
+ logger.info("Initializing ElasticSearch {}", parameterString);
+
+ if (urls != null) {
+ urls = urls.trim();
+ }
+
+ if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls.trim())) {
+ logger.info("Clearing URI config value: {}", urls);
+
+ urls = null;
+ }
+
+ try {
+ if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) {
+ subject = CredentialsProviderUtil.login(user, password);
+ }
+
+ RestClientBuilder restClientBuilder = getRestClientBuilder(urls, protocol, user, password, port);
- private static final Logger logger = LoggerFactory.getLogger(ElasticSearchMgr.class);
- public String index;
- Subject subject;
- String user;
- String password;
-
- synchronized void connect() {
- if (client == null) {
- synchronized (ElasticSearchAuditDestination.class) {
- if (client == null) {
-
- String urls = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_URLS);
- String protocol = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PROTOCOL, "http");
- user = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_USER, "");
- password = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PWRD, "");
- int port = Integer.parseInt(PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PORT));
- this.index = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_INDEX, "ranger_audits");
- String parameterString = String.format(Locale.ROOT,"User:%s, %s://%s:%s/%s", user, protocol, urls, port, index);
- logger.info("Initializing ElasticSearch " + parameterString);
- if (urls != null) {
- urls = urls.trim();
- }
- if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls.trim())) {
- logger.info(String.format("Clearing URI config value: %s", urls));
- urls = null;
- }
-
- try {
- if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) {
- subject = CredentialsProviderUtil.login(user, password);
- }
- RestClientBuilder restClientBuilder =
- getRestClientBuilder(urls, protocol, user, password, port);
- client = new RestHighLevelClient(restClientBuilder);
- } catch (Throwable t) {
- logger.error("Can't connect to ElasticSearch: " + parameterString, t);
- }
- }
- }
- }
- }
-
- public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) {
- RestClientBuilder restClientBuilder = RestClient.builder(
- MiscUtil.toArray(urls, ",").stream()
- .map(x -> new HttpHost(x, port, protocol))
- .<HttpHost>toArray(i -> new HttpHost[i])
- );
- if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) {
- if (password.contains("keytab") && new File(password).exists()) {
- final KerberosCredentialsProvider credentialsProvider =
- CredentialsProviderUtil.getKerberosCredentials(user, password);
- Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
- .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
- restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
- return clientBuilder;
- });
- } else {
- final CredentialsProvider credentialsProvider =
- CredentialsProviderUtil.getBasicCredentials(user, password);
- restClientBuilder.setHttpClientConfigCallback(clientBuilder ->
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider));
- }
- } else {
- logger.error("ElasticSearch Credentials not provided!!");
- final CredentialsProvider credentialsProvider = null;
- restClientBuilder.setHttpClientConfigCallback(clientBuilder ->
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider));
- }
- return restClientBuilder;
- }
-
- RestHighLevelClient client = null;
- public RestHighLevelClient getClient() {
- if (client != null && subject != null) {
- KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject);
- try {
- if (new Date().getTime() > ticket.getEndTime().getTime()){
- client = null;
- CredentialsProviderUtil.ticketExpireTime80 = 0;
- connect();
- } else if (CredentialsProviderUtil.ticketWillExpire(ticket)) {
- subject = CredentialsProviderUtil.login(user, password);
- }
- } catch (PrivilegedActionException e) {
- logger.error("PrivilegedActionException:", e);
- throw new RuntimeException(e);
- }
- return client;
- } else {
- connect();
- }
- return client;
- }
+ client = new RestHighLevelClient(restClientBuilder);
+ } catch (Throwable t) {
+ logger.error("Can't connect to ElasticSearch: {}", parameterString, t);
+ }
+ }
+ }
+ }
+ return me;
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
index 134436667..f9cbcbe43 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
@@ -19,7 +19,11 @@
package org.apache.ranger.elasticsearch;
-import org.apache.ranger.common.*;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.SearchField;
+import org.apache.ranger.common.SortField;
+import org.apache.ranger.common.StringUtil;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.get.MultiGetRequest;
@@ -42,10 +46,11 @@ import org.springframework.stereotype.Component;
import java.io.IOException;
import java.text.SimpleDateFormat;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
@Component
public class ElasticSearchUtil {
@@ -54,17 +59,19 @@ public class ElasticSearchUtil {
@Autowired
StringUtil stringUtil;
- String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'";
- SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr);
+ final String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+ final SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr);
public ElasticSearchUtil() {
String timeZone = PropertiesUtil.getProperty("xa.elasticSearch.timezone");
+
if (timeZone != null) {
- logger.info("Setting timezone to " + timeZone);
+ logger.info("Setting timezone to {}", timeZone);
+
try {
dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
} catch (Throwable t) {
- logger.error("Error setting timezone. TimeZone = " + timeZone);
+ logger.error("Error setting timezone. TimeZone = {}", timeZone);
}
}
}
@@ -72,8 +79,10 @@ public class ElasticSearchUtil {
public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException {
// See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html
QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria);
+
if (searchCriteria.getParamList() != null) {
searchFields.stream().forEach(queryAccumulator::addQuery);
+
// For now assuming there is only date field where range query will
// be done. If we there are more than one, then we should create a
// hashmap for each field name
@@ -81,31 +90,39 @@ public class ElasticSearchUtil {
queryAccumulator.queries.add(setDateRange(queryAccumulator.dateFieldName, queryAccumulator.fromDate, queryAccumulator.toDate));
}
}
+
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+
queryAccumulator.queries.stream().filter(x -> x != null).forEach(boolQueryBuilder::must);
+
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+
setSortClause(searchCriteria, sortFields, searchSourceBuilder);
+
searchSourceBuilder.from(searchCriteria.getStartIndex());
searchSourceBuilder.size(searchCriteria.getMaxRows());
searchSourceBuilder.fetchSource(true);
+
SearchRequest query = new SearchRequest();
+
query.indices(index);
query.source(searchSourceBuilder.query(boolQueryBuilder));
+
return client.search(query, RequestOptions.DEFAULT);
}
- public void setSortClause(SearchCriteria searchCriteria,
- List<SortField> sortFields,
- SearchSourceBuilder searchSourceBuilder) {
-
+ public void setSortClause(SearchCriteria searchCriteria, List<SortField> sortFields, SearchSourceBuilder searchSourceBuilder) {
// TODO: We are supporting single sort field only for now
- String sortBy = searchCriteria.getSortBy();
+ String sortBy = searchCriteria.getSortBy();
String querySortBy = null;
+
if (!stringUtil.isEmpty(sortBy)) {
sortBy = sortBy.trim();
+
for (SortField sortField : sortFields) {
if (sortBy.equalsIgnoreCase(sortField.getParamName())) {
querySortBy = sortField.getFieldName();
+
// Override the sortBy using the normalized value
searchCriteria.setSortBy(sortField.getParamName());
break;
@@ -117,6 +134,7 @@ public class ElasticSearchUtil {
for (SortField sortField : sortFields) {
if (sortField.isDefault()) {
querySortBy = sortField.getFieldName();
+
// Override the sortBy using the default value
searchCriteria.setSortBy(sortField.getParamName());
searchCriteria.setSortType(sortField.getDefaultOrder().name());
@@ -127,11 +145,13 @@ public class ElasticSearchUtil {
if (querySortBy != null) {
// Add sort type
- String sortType = searchCriteria.getSortType();
- SortOrder order = SortOrder.ASC;
- if (sortType != null && "desc".equalsIgnoreCase(sortType)) {
+ String sortType = searchCriteria.getSortType();
+ SortOrder order = SortOrder.ASC;
+
+ if ("desc".equalsIgnoreCase(sortType)) {
order = SortOrder.DESC;
}
+
searchSourceBuilder.sort(querySortBy, order);
}
}
@@ -140,6 +160,7 @@ public class ElasticSearchUtil {
if (valueList == null || valueList.isEmpty()) {
return null;
}
+
if (valueList.isEmpty()) {
return null;
} else {
@@ -152,63 +173,73 @@ public class ElasticSearchUtil {
}
}
-
- private String filterText(Object value) {
- return ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase());
- }
-
public QueryBuilder setDateRange(String fieldName, Date fromDate, Date toDate) {
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(fieldName).format(dateFormateStr);
+
if (fromDate != null) {
rangeQueryBuilder.from(dateFormat.format(fromDate));
}
+
if (toDate != null) {
rangeQueryBuilder.to(dateFormat.format(toDate));
}
+
return rangeQueryBuilder;
}
public MultiGetItemResponse[] fetch(RestHighLevelClient client, String index, SearchHit... hits) throws IOException {
- if(0 == hits.length) {
+ if (0 == hits.length) {
return new MultiGetItemResponse[0];
}
+
MultiGetRequest multiGetRequest = new MultiGetRequest();
+
for (SearchHit hit : hits) {
MultiGetRequest.Item item = new MultiGetRequest.Item(index, null, hit.getId());
+
item.fetchSourceContext(FetchSourceContext.FETCH_SOURCE);
+
multiGetRequest.add(item);
}
+
return client.multiGet(multiGetRequest, RequestOptions.DEFAULT).getResponses();
}
+ private String filterText(Object value) {
+ return ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase());
+ }
+
private class QueryAccumulator {
public final List<QueryBuilder> queries = new ArrayList<>();
- public final SearchCriteria searchCriteria;
- public Date fromDate;
- public Date toDate;
- public String dateFieldName;
+ public final SearchCriteria searchCriteria;
+ public Date fromDate;
+ public Date toDate;
+ public String dateFieldName;
private QueryAccumulator(SearchCriteria searchCriteria) {
this.searchCriteria = searchCriteria;
- this.fromDate = null;
- this.toDate = null;
- this.dateFieldName = null;
+ this.fromDate = null;
+ this.toDate = null;
+ this.dateFieldName = null;
}
public QueryAccumulator addQuery(SearchField searchField) {
QueryBuilder queryBuilder = getQueryBuilder(searchField);
+
if (null != queryBuilder) {
queries.add(queryBuilder);
}
+
return this;
}
public QueryBuilder getQueryBuilder(SearchField searchField) {
- String clientFieldName = searchField.getClientFieldName();
- String fieldName = searchField.getFieldName();
- SearchField.DATA_TYPE dataType = searchField.getDataType();
- SearchField.SEARCH_TYPE searchType = searchField.getSearchType();
- Object paramValue = searchCriteria.getParamValue(clientFieldName);
+ String clientFieldName = searchField.getClientFieldName();
+ String fieldName = searchField.getFieldName();
+ SearchField.DATA_TYPE dataType = searchField.getDataType();
+ SearchField.SEARCH_TYPE searchType = searchField.getSearchType();
+ Object paramValue = searchCriteria.getParamValue(clientFieldName);
+
return getQueryBuilder(dataType, searchType, fieldName, paramValue);
}
@@ -216,12 +247,15 @@ public class ElasticSearchUtil {
if (paramValue == null || paramValue.toString().isEmpty()) {
return null;
}
+
if (fieldName.startsWith("-")) {
QueryBuilder negativeQuery = getQueryBuilder(dataType, searchType, fieldName.substring(1), paramValue);
return null == negativeQuery ? null : QueryBuilders.boolQuery().mustNot(negativeQuery);
}
+
if (paramValue instanceof Collection) {
Collection<?> valueList = (Collection<?>) paramValue;
+
if (valueList.isEmpty()) {
return null;
} else {
@@ -235,27 +269,22 @@ public class ElasticSearchUtil {
} else {
if (dataType == SearchField.DATA_TYPE.DATE) {
if (!(paramValue instanceof Date)) {
- logger.error(String.format(
- "Search value is not a Java Date Object: %s %s %s",
- fieldName, searchType, paramValue));
+ logger.error("Search value is not a Java Date Object: {} {} {}", fieldName, searchType, paramValue);
} else {
- if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN
- || searchType == SearchField.SEARCH_TYPE.GREATER_THAN) {
- fromDate = (Date) paramValue;
+ if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.GREATER_THAN) {
+ fromDate = (Date) paramValue;
dateFieldName = fieldName;
- } else if (searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN
- || searchType == SearchField.SEARCH_TYPE.LESS_THAN) {
- toDate = (Date) paramValue;
+ } else if (searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.LESS_THAN) {
+ toDate = (Date) paramValue;
dateFieldName = fieldName;
}
}
+
return null;
- } else if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN
- || searchType == SearchField.SEARCH_TYPE.GREATER_THAN
- || searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN
- || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { //NOPMD
- logger.warn(String.format("Range Queries Not Implemented: %s %s %s",
- fieldName, searchType, paramValue));
+ } else if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.GREATER_THAN
+ || searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { //NOPMD
+ logger.warn("Range Queries Not Implemented: {} {} {}", fieldName, searchType, paramValue);
+
return null;
} else {
if (searchType == SearchField.SEARCH_TYPE.PARTIAL) {
@@ -274,6 +303,5 @@ public class ElasticSearchUtil {
}
}
}
-
}
}