You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2020/05/27 08:32:49 UTC
[ranger] branch master updated: RANGER-2634: Add support for
ElasticSearch as an Audit Database
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new cf1e3ef RANGER-2634: Add support for ElasticSearch as an Audit Database
cf1e3ef is described below
commit cf1e3ef8dcd5a4adc046a3895f49c9deb3069015
Author: acharneski@gmail.com <ac...@gmail.com>
AuthorDate: Sun May 24 14:43:40 2020 +0530
RANGER-2634: Add support for ElasticSearch as an Audit Database
Signed-off-by: pradeep <pr...@apache.org>
---
agents-audit/pom.xml | 63 +++-
.../destination/ElasticSearchAuditDestination.java | 273 +++++++++++++++++
.../audit/provider/AuditProviderFactory.java | 44 ++-
hbase-agent/conf/ranger-hbase-audit-changes.cfg | 8 +
hbase-agent/conf/ranger-hbase-audit.xml | 6 +-
hbase-agent/scripts/install.properties | 12 +
hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 8 +
hdfs-agent/conf/ranger-hdfs-audit.xml | 6 +-
hdfs-agent/scripts/install.properties | 12 +
hive-agent/conf/ranger-hive-audit-changes.cfg | 8 +
hive-agent/conf/ranger-hive-audit.xml | 6 +-
hive-agent/scripts/install.properties | 12 +
kms/scripts/install.properties | 12 +
knox-agent/conf/ranger-knox-audit-changes.cfg | 8 +
knox-agent/conf/ranger-knox-audit.xml | 6 +-
knox-agent/scripts/install.properties | 12 +
plugin-atlas/conf/ranger-atlas-audit-changes.cfg | 5 +
plugin-atlas/conf/ranger-atlas-audit.xml | 17 +-
plugin-atlas/scripts/install.properties | 12 +
.../conf/ranger-elasticsearch-audit-changes.cfg | 8 +
.../conf/ranger-elasticsearch-audit.xml | 4 +
plugin-elasticsearch/scripts/install.properties | 12 +
plugin-kafka/conf/ranger-kafka-audit-changes.cfg | 8 +
plugin-kafka/conf/ranger-kafka-audit.xml | 6 +-
plugin-kafka/scripts/install.properties | 12 +
plugin-kms/conf/ranger-kms-audit-changes.cfg | 8 +
plugin-kms/conf/ranger-kms-audit.xml | 6 +-
plugin-kylin/conf/ranger-kylin-audit-changes.cfg | 8 +
plugin-kylin/conf/ranger-kylin-audit.xml | 4 +
plugin-kylin/scripts/install.properties | 12 +
plugin-ozone/conf/ranger-ozone-audit-changes.cfg | 8 +
plugin-ozone/conf/ranger-ozone-audit.xml | 4 +
plugin-ozone/scripts/install.properties | 12 +
plugin-presto/conf/ranger-presto-audit-changes.cfg | 8 +
plugin-presto/conf/ranger-presto-audit.xml | 4 +
plugin-presto/scripts/install.properties | 12 +
plugin-solr/conf/ranger-solr-audit-changes.cfg | 8 +
plugin-solr/conf/ranger-solr-audit.xml | 6 +-
plugin-solr/scripts/install.properties | 12 +
plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg | 8 +
plugin-sqoop/conf/ranger-sqoop-audit.xml | 4 +
plugin-sqoop/scripts/install.properties | 12 +
plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 8 +
plugin-yarn/conf/ranger-yarn-audit.xml | 8 +-
plugin-yarn/scripts/install.properties | 12 +
pom.xml | 2 +
.../plugin/RangerElasticsearchPlugin.java | 2 +-
.../contrib/es_for_audit_setup/README.txt | 22 ++
.../contrib/es_for_audit_setup/create_index.sh | 20 ++
.../contrib/es_for_audit_setup/enable_auth.sh | 26 ++
.../contrib/es_for_audit_setup/install_es.sh | 22 ++
security-admin/scripts/install.properties | 8 +-
.../scripts/ranger-admin-site-template.xml | 16 +
security-admin/scripts/setup.sh | 33 ++
security-admin/scripts/upgrade_admin.py | 4 +
.../org/apache/ranger/AccessAuditsService.java | 155 ++++++++++
.../main/java/org/apache/ranger/biz/AssetMgr.java | 6 +
.../java/org/apache/ranger/biz/RangerBizUtil.java | 1 +
.../main/java/org/apache/ranger/biz/XAuditMgr.java | 14 +-
.../ElasticSearchAccessAuditsService.java | 283 ++++++++++++++++++
.../ranger/elasticsearch/ElasticSearchMgr.java | 106 +++++++
.../ranger/elasticsearch/ElasticSearchUtil.java | 331 +++++++++++++++++++++
.../ranger/solr/SolrAccessAuditsService.java | 134 +--------
.../main/resources/conf.dist/ranger-admin-site.xml | 20 ++
.../ElasticSearchAccessAuditsServiceTest.java | 139 +++++++++
storm-agent/conf/ranger-storm-audit-changes.cfg | 8 +
storm-agent/conf/ranger-storm-audit.xml | 6 +-
storm-agent/scripts/install.properties | 12 +
68 files changed, 1950 insertions(+), 164 deletions(-)
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index 8ac1edf..12d621f 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -23,6 +23,7 @@
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <securesm.version>1.2</securesm.version>
</properties>
<parent>
<groupId>org.apache.ranger</groupId>
@@ -85,7 +86,7 @@
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
- <version>${solr.version}</version>
+ <version>${solr.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -102,6 +103,66 @@
<artifactId>guava</artifactId>
<version>${google.guava.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-core</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>securesm</artifactId>
+ <version>${securesm.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>parent-join-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>aggs-matrix-stats-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>rank-eval-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>lang-mustache-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore-nio</artifactId>
+ <version>${httpcomponents.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ <version>${httpcomponents.httpasyncclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
new file mode 100644
index 0000000..dc00fec
--- /dev/null
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.destination;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+
+
+public class ElasticSearchAuditDestination extends AuditDestination {
+ private static final Log LOG = LogFactory.getLog(ElasticSearchAuditDestination.class);
+
+ public static final String CONFIG_URLS = "urls";
+ public static final String CONFIG_PORT = "port";
+ public static final String CONFIG_USER = "user";
+ public static final String CONFIG_PWRD = "password";
+ public static final String CONFIG_PROTOCOL = "protocol";
+ public static final String CONFIG_INDEX = "index";
+ public static final String CONFIG_PREFIX = "ranger.audit.elasticsearch";
+ public static final String DEFAULT_INDEX = "audit";
+
+ private String index = "index";
+ private volatile RestHighLevelClient client = null;
+ private String protocol;
+ private String user;
+ private int port;
+ private String password;
+ private String hosts;
+
+ public ElasticSearchAuditDestination() {
+ propPrefix = CONFIG_PREFIX;
+ }
+
+
+ @Override
+ public void init(Properties props, String propPrefix) {
+ super.init(props, propPrefix);
+ this.protocol = getStringProperty(props, propPrefix + "." + CONFIG_PROTOCOL, "http");
+ this.user = getStringProperty(props, propPrefix + "." + CONFIG_USER, "");
+ this.password = getStringProperty(props, propPrefix + "." + CONFIG_PWRD, "");
+ this.port = MiscUtil.getIntProperty(props, propPrefix + "." + CONFIG_PORT, 9200);
+ this.index = getStringProperty(props, propPrefix + "." + CONFIG_INDEX, DEFAULT_INDEX);
+ this.hosts = getHosts();
+ LOG.info("Connecting to ElasticSearch: " + connectionString());
+ getClient(); // Initialize client
+ }
+
+ private String connectionString() {
+ return String.format("%s://%s@%s:%s/%s", protocol, user, hosts, port, index);
+ }
+
+ @Override
+ public void stop() {
+ super.stop();
+ logStatus();
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ boolean ret = false;
+ try {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ RestHighLevelClient client = getClient();
+ if (null == client) {
+ // ElasticSearch is still not initialized. So need return error
+ addDeferredCount(events.size());
+ return ret;
+ }
+
+ ArrayList<AuditEventBase> eventList = new ArrayList<>(events);
+ BulkRequest bulkRequest = new BulkRequest();
+ try {
+ for (AuditEventBase event : eventList) {
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+ String id = authzEvent.getEventId();
+ Map<String, Object> doc = toDoc(authzEvent);
+ bulkRequest.add(new IndexRequest(index).id(id).source(doc).type(""));
+ }
+ } catch (Exception ex) {
+ addFailedCount(eventList.size());
+ logFailedEvent(eventList, ex);
+ }
+ BulkResponse response = client.bulk(bulkRequest);
+ if (response.status().getStatus() >= 400) {
+ addFailedCount(eventList.size());
+ logFailedEvent(eventList, "HTTP " + response.status().getStatus());
+ } else {
+ BulkItemResponse[] items = response.getItems();
+ for (int i = 0; i < items.length; i++) {
+ AuditEventBase itemRequest = eventList.get(i);
+ BulkItemResponse itemResponse = items[i];
+ if (itemResponse.isFailed()) {
+ addFailedCount(1);
+ logFailedEvent(Arrays.asList(itemRequest), itemResponse.getFailureMessage());
+ } else {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Indexed %s", itemRequest.getEventKey()));
+ }
+ addSuccessCount(1);
+ ret = true;
+ }
+ }
+ }
+ } catch (Throwable t) {
+ addDeferredCount(events.size());
+ logError("Error sending message to ElasticSearch", t);
+ }
+ return ret;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+ synchronized RestHighLevelClient getClient() {
+ if (client == null) {
+ synchronized (ElasticSearchAuditDestination.class) {
+ if (client == null) {
+ client = newClient();
+ }
+ }
+ }
+ return client;
+ }
+
+ private final AtomicLong lastLoggedAt = new AtomicLong(0);
+
+ private RestHighLevelClient newClient() {
+ try {
+ final CredentialsProvider credentialsProvider;
+ if(!user.isEmpty()) {
+ credentialsProvider = new BasicCredentialsProvider();
+ credentialsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(user, password));
+ } else {
+ credentialsProvider = null;
+ }
+
+ RestHighLevelClient restHighLevelClient = new RestHighLevelClient(
+ RestClient.builder(
+ MiscUtil.toArray(hosts, ",").stream()
+ .map(x -> new HttpHost(x, port, protocol))
+ .<HttpHost>toArray(i -> new HttpHost[i])
+ ).setHttpClientConfigCallback(clientBuilder ->
+ (credentialsProvider != null) ? clientBuilder.setDefaultCredentialsProvider(credentialsProvider) : clientBuilder));
+ LOG.debug("Initialized client");
+ boolean exits = false;
+ try {
+ exits = restHighLevelClient.indices().open(new OpenIndexRequest(this.index)).isShardsAcknowledged();
+ } catch (Exception e) {
+ LOG.warn("Error validating index " + this.index);
+ }
+ if(exits) {
+ LOG.debug("Index exists");
+ } else {
+ LOG.info("Index does not exist");
+ }
+ return restHighLevelClient;
+ } catch (Throwable t) {
+ lastLoggedAt.updateAndGet(lastLoggedAt -> {
+ long now = System.currentTimeMillis();
+ long elapsed = now - lastLoggedAt;
+ if (elapsed > TimeUnit.MINUTES.toMillis(1)) {
+ LOG.fatal("Can't connect to ElasticSearch server: " + connectionString(), t);
+ return now;
+ } else {
+ return lastLoggedAt;
+ }
+ });
+ return null;
+ }
+ }
+
+ private String getHosts() {
+ String urls = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_URLS);
+ if (urls != null) {
+ urls = urls.trim();
+ }
+ if (urls != null && urls.equalsIgnoreCase("NONE")) {
+ urls = null;
+ }
+ return urls;
+ }
+
+ private String getStringProperty(Properties props, String propName, String defaultValue) {
+ String value = MiscUtil.getStringProperty(props, propName);
+ if (null == value) return defaultValue;
+ return value;
+ }
+
+ Map<String, Object> toDoc(AuthzAuditEvent auditEvent) {
+ Map<String, Object> doc = new HashMap<String, Object>();
+ doc.put("id", auditEvent.getEventId());
+ doc.put("access", auditEvent.getAccessType());
+ doc.put("enforcer", auditEvent.getAclEnforcer());
+ doc.put("agent", auditEvent.getAgentId());
+ doc.put("repo", auditEvent.getRepositoryName());
+ doc.put("sess", auditEvent.getSessionId());
+ doc.put("reqUser", auditEvent.getUser());
+ doc.put("reqData", auditEvent.getRequestData());
+ doc.put("resource", auditEvent.getResourcePath());
+ doc.put("cliIP", auditEvent.getClientIP());
+ doc.put("logType", auditEvent.getLogType());
+ doc.put("result", auditEvent.getAccessResult());
+ doc.put("policy", auditEvent.getPolicyId());
+ doc.put("repoType", auditEvent.getRepositoryType());
+ doc.put("resType", auditEvent.getResourceType());
+ doc.put("reason", auditEvent.getResultReason());
+ doc.put("action", auditEvent.getAction());
+ doc.put("evtTime", auditEvent.getEventTime());
+ doc.put("seq_num", auditEvent.getSeqNum());
+ doc.put("event_count", auditEvent.getEventCount());
+ doc.put("event_dur_ms", auditEvent.getEventDurationMS());
+ doc.put("tags", auditEvent.getTags());
+ doc.put("cluster", auditEvent.getClusterName());
+ doc.put("zoneName", auditEvent.getZoneName());
+ doc.put("agentHost", auditEvent.getAgentHostname());
+ doc.put("policyVersion", auditEvent.getPolicyVersion());
+ return doc;
+ }
+
+}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index 88cf99b..8ee84ac 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -28,11 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.ranger.audit.destination.DBAuditDestination;
-import org.apache.ranger.audit.destination.FileAuditDestination;
-import org.apache.ranger.audit.destination.HDFSAuditDestination;
-import org.apache.ranger.audit.destination.Log4JAuditDestination;
-import org.apache.ranger.audit.destination.SolrAuditDestination;
+import org.apache.ranger.audit.destination.*;
import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider;
import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider;
import org.apache.ranger.audit.provider.solr.SolrAuditProvider;
@@ -58,6 +54,7 @@ public class AuditProviderFactory {
public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled";
public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
+ public static final String AUDIT_ELASTICSEARCH_IS_ENABLED_PROP = "xasecure.audit.elasticsearch.is.enabled";
public static final String AUDIT_DEST_BASE = "xasecure.audit.destination";
public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds";
@@ -142,6 +139,8 @@ public class AuditProviderFactory {
AUDIT_KAFKA_IS_ENABLED_PROP, false);
boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props,
AUDIT_SOLR_IS_ENABLED_PROP, false);
+ boolean isAuditToElasticsearchEnabled = MiscUtil.getBooleanProperty(props,
+ AUDIT_ELASTICSEARCH_IS_ENABLED_PROP, false);
boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props,
AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false);
@@ -283,8 +282,9 @@ public class AuditProviderFactory {
LOG.info("No v3 audit configuration found. Trying v2 audit configurations");
if (!isEnabled
|| !(isAuditToDbEnabled || isAuditToHdfsEnabled
- || isAuditToKafkaEnabled || isAuditToLog4jEnabled
- || isAuditToSolrEnabled || providers.size() == 0)) {
+ || isAuditToKafkaEnabled || isAuditToLog4jEnabled
+ || isAuditToSolrEnabled || isAuditToElasticsearchEnabled
+ || providers.size() == 0)) {
LOG.info("AuditProviderFactory: Audit not enabled..");
mProvider = getDefaultProvider();
@@ -373,6 +373,20 @@ public class AuditProviderFactory {
}
}
+ if (isAuditToElasticsearchEnabled) {
+ LOG.info("ElasticsearchAuditProvider is enabled");
+ ElasticSearchAuditDestination elasticSearchAuditDestination = new ElasticSearchAuditDestination();
+ elasticSearchAuditDestination.init(props);
+
+ if (elasticSearchAuditDestination.isAsync()) {
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider(
+ "MyElasticSearchAuditProvider", 1000, 1000, elasticSearchAuditDestination);
+ providers.add(asyncProvider);
+ } else {
+ providers.add(elasticSearchAuditDestination);
+ }
+ }
+
if (isAuditToLog4jEnabled) {
Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
@@ -440,21 +454,23 @@ public class AuditProviderFactory {
+ ", propertyPrefix=" + propPrefix, e);
}
} else {
- if (providerName.equals("file")) {
+ if (providerName.equalsIgnoreCase("file")) {
provider = new FileAuditDestination();
} else if (providerName.equalsIgnoreCase("hdfs")) {
provider = new HDFSAuditDestination();
- } else if (providerName.equals("solr")) {
+ } else if (providerName.equalsIgnoreCase("solr")) {
provider = new SolrAuditDestination();
- } else if (providerName.equals("kafka")) {
+ } else if (providerName.equalsIgnoreCase("elasticsearch")) {
+ provider = new ElasticSearchAuditDestination();
+ } else if (providerName.equalsIgnoreCase("kafka")) {
provider = new KafkaAuditProvider();
- } else if (providerName.equals("db")) {
+ } else if (providerName.equalsIgnoreCase("db")) {
provider = new DBAuditDestination();
- } else if (providerName.equals("log4j")) {
+ } else if (providerName.equalsIgnoreCase("log4j")) {
provider = new Log4JAuditDestination();
- } else if (providerName.equals("batch")) {
+ } else if (providerName.equalsIgnoreCase("batch")) {
provider = new AuditBatchQueue(consumer);
- } else if (providerName.equals("async")) {
+ } else if (providerName.equalsIgnoreCase("async")) {
provider = new AuditAsyncQueue(consumer);
} else {
LOG.error("Provider name doesn't have any class associated with it. providerName="
diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
index 719c7cd..084e3c1 100644
--- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg
+++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
@@ -54,6 +54,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml
index 5f88224..f558ee6 100644
--- a/hbase-agent/conf/ranger-hbase-audit.xml
+++ b/hbase-agent/conf/ranger-hbase-audit.xml
@@ -250,7 +250,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index f4fdb14..7311669 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -56,6 +56,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hbase/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
index e34d154..3d13626 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
+++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
@@ -51,6 +51,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml
index a380906..5e35259 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit.xml
+++ b/hdfs-agent/conf/ranger-hdfs-audit.xml
@@ -250,7 +250,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 54dc3a1..f5e99fd 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/hdfs/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg
index 3fd7e14..611a96e 100644
--- a/hive-agent/conf/ranger-hive-audit-changes.cfg
+++ b/hive-agent/conf/ranger-hive-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml
index ae1dce9..2ea8ef6 100644
--- a/hive-agent/conf/ranger-hive-audit.xml
+++ b/hive-agent/conf/ranger-hive-audit.xml
@@ -250,7 +250,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 3e8f590..223984d 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -53,6 +53,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties
index a30b1d3..4d82121 100755
--- a/kms/scripts/install.properties
+++ b/kms/scripts/install.properties
@@ -163,6 +163,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/ranger/kms/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg
index f0571e7..0eb91ef 100644
--- a/knox-agent/conf/ranger-knox-audit-changes.cfg
+++ b/knox-agent/conf/ranger-knox-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml
index 7b5bfb3..537a226 100644
--- a/knox-agent/conf/ranger-knox-audit.xml
+++ b/knox-agent/conf/ranger-knox-audit.xml
@@ -250,7 +250,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index d2dbbc3..a967bb2 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -48,6 +48,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/knox/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg
index 07fc382..e73b495 100644
--- a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg
+++ b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg
@@ -21,6 +21,11 @@ xasecure.audit.destination.solr.user %XAAUDIT.SOLR.USER% mod
xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-not-exists
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
#hdfs configuration
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
diff --git a/plugin-atlas/conf/ranger-atlas-audit.xml b/plugin-atlas/conf/ranger-atlas-audit.xml
index 93ad238..9f03227 100644
--- a/plugin-atlas/conf/ranger-atlas-audit.xml
+++ b/plugin-atlas/conf/ranger-atlas-audit.xml
@@ -42,9 +42,20 @@
<property>
<name>xasecure.audit.destination.solr.collection</name>
<value>NONE</value>
- </property>
-
-
+ </property>
+
+ <!-- ElasticSearch audit provider configuration -->
+ <property>
+ <name>xasecure.audit.destination.elasticsearch</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.destination.elasticsearch.urls</name>
+ <value>NONE</value>
+ </property>
+
+
<!-- HDFS audit provider configuration -->
<property>
<name>xasecure.audit.destination.hdfs</name>
diff --git a/plugin-atlas/scripts/install.properties b/plugin-atlas/scripts/install.properties
index 511e6ae..ce952d5 100644
--- a/plugin-atlas/scripts/install.properties
+++ b/plugin-atlas/scripts/install.properties
@@ -52,6 +52,18 @@ XAAUDIT.SOLR.USER=NONE
XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
index 8071e7b..39d7955 100644
--- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
+++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit.xml b/plugin-elasticsearch/conf/ranger-elasticsearch-audit.xml
index b9bdde5..e9934ff 100644
--- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit.xml
+++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit.xml
@@ -252,6 +252,10 @@
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-elasticsearch/scripts/install.properties b/plugin-elasticsearch/scripts/install.properties
index 3a5b213..2a22f54 100644
--- a/plugin-elasticsearch/scripts/install.properties
+++ b/plugin-elasticsearch/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/elasticsearch/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
index 661b498..133d3a7 100644
--- a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
+++ b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
@@ -45,6 +45,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-kafka/conf/ranger-kafka-audit.xml b/plugin-kafka/conf/ranger-kafka-audit.xml
index 5fbbf33..93b0fc5 100644
--- a/plugin-kafka/conf/ranger-kafka-audit.xml
+++ b/plugin-kafka/conf/ranger-kafka-audit.xml
@@ -252,7 +252,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-kafka/scripts/install.properties b/plugin-kafka/scripts/install.properties
index 6b01aed..3764c3d 100644
--- a/plugin-kafka/scripts/install.properties
+++ b/plugin-kafka/scripts/install.properties
@@ -51,6 +51,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/kafka/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-kms/conf/ranger-kms-audit-changes.cfg b/plugin-kms/conf/ranger-kms-audit-changes.cfg
index 69849d6..b140623 100644
--- a/plugin-kms/conf/ranger-kms-audit-changes.cfg
+++ b/plugin-kms/conf/ranger-kms-audit-changes.cfg
@@ -54,6 +54,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-kms/conf/ranger-kms-audit.xml b/plugin-kms/conf/ranger-kms-audit.xml
index 02c1a79..d1a99f7 100755
--- a/plugin-kms/conf/ranger-kms-audit.xml
+++ b/plugin-kms/conf/ranger-kms-audit.xml
@@ -249,7 +249,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg
index 8071e7b..39d7955 100644
--- a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg
+++ b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-kylin/conf/ranger-kylin-audit.xml b/plugin-kylin/conf/ranger-kylin-audit.xml
index 94fba58..b977bc5 100644
--- a/plugin-kylin/conf/ranger-kylin-audit.xml
+++ b/plugin-kylin/conf/ranger-kylin-audit.xml
@@ -252,6 +252,10 @@
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-kylin/scripts/install.properties b/plugin-kylin/scripts/install.properties
index 126eeba..0a8c5e6 100644
--- a/plugin-kylin/scripts/install.properties
+++ b/plugin-kylin/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/kylin/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg
index e5adb76..c022c06 100644
--- a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg
+++ b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg
@@ -45,6 +45,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-ozone/conf/ranger-ozone-audit.xml b/plugin-ozone/conf/ranger-ozone-audit.xml
index 05a18d6..1808f37 100644
--- a/plugin-ozone/conf/ranger-ozone-audit.xml
+++ b/plugin-ozone/conf/ranger-ozone-audit.xml
@@ -243,6 +243,10 @@
<value>false</value>
</property>
<property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
<name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
diff --git a/plugin-ozone/scripts/install.properties b/plugin-ozone/scripts/install.properties
index 276d192..ac11749 100644
--- a/plugin-ozone/scripts/install.properties
+++ b/plugin-ozone/scripts/install.properties
@@ -51,6 +51,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/ozone/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-presto/conf/ranger-presto-audit-changes.cfg b/plugin-presto/conf/ranger-presto-audit-changes.cfg
index 661b498..133d3a7 100644
--- a/plugin-presto/conf/ranger-presto-audit-changes.cfg
+++ b/plugin-presto/conf/ranger-presto-audit-changes.cfg
@@ -45,6 +45,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-presto/conf/ranger-presto-audit.xml b/plugin-presto/conf/ranger-presto-audit.xml
index c72771e..6985f92 100644
--- a/plugin-presto/conf/ranger-presto-audit.xml
+++ b/plugin-presto/conf/ranger-presto-audit.xml
@@ -253,6 +253,10 @@
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-presto/scripts/install.properties b/plugin-presto/scripts/install.properties
index 3110e2d..1ead4fe 100644
--- a/plugin-presto/scripts/install.properties
+++ b/plugin-presto/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/presto/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-solr/conf/ranger-solr-audit-changes.cfg b/plugin-solr/conf/ranger-solr-audit-changes.cfg
index 622052e..aad786a 100644
--- a/plugin-solr/conf/ranger-solr-audit-changes.cfg
+++ b/plugin-solr/conf/ranger-solr-audit-changes.cfg
@@ -46,6 +46,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-solr/conf/ranger-solr-audit.xml b/plugin-solr/conf/ranger-solr-audit.xml
index f55b623..aa10e34 100644
--- a/plugin-solr/conf/ranger-solr-audit.xml
+++ b/plugin-solr/conf/ranger-solr-audit.xml
@@ -252,7 +252,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-solr/scripts/install.properties b/plugin-solr/scripts/install.properties
index 48a9af2..49533c1 100644
--- a/plugin-solr/scripts/install.properties
+++ b/plugin-solr/scripts/install.properties
@@ -51,6 +51,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/solr/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg
index 8071e7b..39d7955 100644
--- a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg
+++ b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-sqoop/conf/ranger-sqoop-audit.xml b/plugin-sqoop/conf/ranger-sqoop-audit.xml
index 013a84c..a84514f 100644
--- a/plugin-sqoop/conf/ranger-sqoop-audit.xml
+++ b/plugin-sqoop/conf/ranger-sqoop-audit.xml
@@ -252,6 +252,10 @@
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/plugin-sqoop/scripts/install.properties b/plugin-sqoop/scripts/install.properties
index 44f16da..4dbc973 100644
--- a/plugin-sqoop/scripts/install.properties
+++ b/plugin-sqoop/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/sqoop/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
index 8071e7b..39d7955 100644
--- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
+++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml
index 667419d..1a49e9e 100644
--- a/plugin-yarn/conf/ranger-yarn-audit.xml
+++ b/plugin-yarn/conf/ranger-yarn-audit.xml
@@ -251,8 +251,12 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
-
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
+
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties
index f776c5f..a5ae529 100644
--- a/plugin-yarn/scripts/install.properties
+++ b/plugin-yarn/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hadoop/yarn/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true
diff --git a/pom.xml b/pom.xml
index 97677ce..5605a17 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,6 +133,7 @@
<htrace-core.version>4.1.0-incubating</htrace-core.version>
<httpcomponents.httpclient.version>4.5.6</httpcomponents.httpclient.version>
<httpcomponents.httpcore.version>4.4.6</httpcomponents.httpcore.version>
+ <httpcomponents.httpasyncclient.version>4.1.3</httpcomponents.httpasyncclient.version>
<httpcomponents.httpmime.version>4.5.6</httpcomponents.httpmime.version>
<javax.persistence.version>2.1.0</javax.persistence.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
@@ -216,6 +217,7 @@
<net.minidev.asm.version>1.0.2</net.minidev.asm.version>
<org.bouncycastle.bcprov-jdk15on>1.59</org.bouncycastle.bcprov-jdk15on>
<org.bouncycastle.bcpkix-jdk15on>1.59</org.bouncycastle.bcpkix-jdk15on>
+ <lucene.version>7.2.1</lucene.version>
</properties>
<profiles>
<profile>
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
index f9a6837..faeac52 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
@@ -101,7 +101,7 @@ public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin {
try {
if (configFile.exists()) {
ClassLoader classLoader = this.getClass().getClassLoader();
- // This classLoader is FactoryURLClassLoader in eleasticsearch
+ // This classLoader is FactoryURLClassLoader in elasticsearch
if (classLoader instanceof URLClassLoader) {
URLClassLoader urlClassLoader = (URLClassLoader) classLoader;
Class<? extends URLClassLoader> urlClass = urlClassLoader.getClass();
diff --git a/security-admin/contrib/es_for_audit_setup/README.txt b/security-admin/contrib/es_for_audit_setup/README.txt
new file mode 100644
index 0000000..badbb22
--- /dev/null
+++ b/security-admin/contrib/es_for_audit_setup/README.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+This folder consists of the scripts required to install and configure
+Elasticsearch as an audit destination/source for Apache Ranger.
+
+Scripts:
+1. install_es.sh - Downloads and installs the basic single-node elasticsearch server
+2. sudo enable_auth.sh - Configures elasticsearch to use password authentication
+3. create_index.sh - Creates and configures the index
diff --git a/security-admin/contrib/es_for_audit_setup/create_index.sh b/security-admin/contrib/es_for_audit_setup/create_index.sh
new file mode 100644
index 0000000..9f21a88
--- /dev/null
+++ b/security-admin/contrib/es_for_audit_setup/create_index.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a new elasticsearch index
+# and configures it with Ranger Audit Log fields
+
+curl --user elastic -X PUT "localhost:9200/audit?pretty"
diff --git a/security-admin/contrib/es_for_audit_setup/enable_auth.sh b/security-admin/contrib/es_for_audit_setup/enable_auth.sh
new file mode 100644
index 0000000..9c256b7
--- /dev/null
+++ b/security-admin/contrib/es_for_audit_setup/enable_auth.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script enables authentication for the single-node local elasticsearch installation.
+# It will start another script to setup each elasticsearh service password.
+
+service elasticsearch stop
+cat >> /etc/elasticsearch/elasticsearch.yml << EOF
+xpack.security.enabled: true
+EOF
+service elasticsearch start
+sleep 10
+/usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive
diff --git a/security-admin/contrib/es_for_audit_setup/install_es.sh b/security-admin/contrib/es_for_audit_setup/install_es.sh
new file mode 100644
index 0000000..5463eb1
--- /dev/null
+++ b/security-admin/contrib/es_for_audit_setup/install_es.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script downloads and installs Elasticsearch (single node)
+# This requires RPM, but similar downloads exist on Elasticsearch's site for tar or deb
+
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.5.1-x86_64.rpm
+sudo rpm --install elasticsearch-7.5.1-x86_64.rpm
+sudo -i service elasticsearch start
diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties
index 155c42c..45e0a94 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -77,10 +77,16 @@ rangerUsersync_password=
keyadmin_password=
-#Source for Audit Store. Currently only solr is supported.
+#Source for Audit Store. Currently solr and elasticsearch are supported.
# * audit_store is solr
audit_store=solr
+# * audit_solr_url Elasticsearch Host(s). E.g. 127.0.0.1
+audit_elasticsearch_urls=
+audit_elasticsearch_port=
+audit_elasticsearch_user=
+audit_elasticsearch_password=
+
# * audit_solr_url URL to Solr. E.g. http://<solr_host>:6083/solr/ranger_audits
audit_solr_urls=
audit_solr_user=
diff --git a/security-admin/scripts/ranger-admin-site-template.xml b/security-admin/scripts/ranger-admin-site-template.xml
index af345cf..72ff66e 100644
--- a/security-admin/scripts/ranger-admin-site-template.xml
+++ b/security-admin/scripts/ranger-admin-site-template.xml
@@ -161,6 +161,22 @@
<value></value>
</property>
<property>
+ <name>ranger.audit.elasticsearch.urls</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.elasticsearch.port</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.elasticsearch.user</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.elasticsearch.password</name>
+ <value></value>
+ </property>
+ <property>
<name>ranger.jpa.audit.jdbc.dialect</name>
<value></value>
</property>
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index 9677f57..61d351d 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -72,6 +72,10 @@ javax_net_ssl_keyStorePassword=$(get_prop 'javax_net_ssl_keyStorePassword' $PROP
javax_net_ssl_trustStore=$(get_prop 'javax_net_ssl_trustStore' $PROPFILE)
javax_net_ssl_trustStorePassword=$(get_prop 'javax_net_ssl_trustStorePassword' $PROPFILE)
audit_store=$(get_prop 'audit_store' $PROPFILE)
+audit_elasticsearch_urls=$(get_prop 'audit_elasticsearch_urls' $PROPFILE)
+audit_elasticsearch_port=$(get_prop 'audit_elasticsearch_port' $PROPFILE)
+audit_elasticsearch_user=$(get_prop 'audit_elasticsearch_user' $PROPFILE)
+audit_elasticsearch_password=$(get_prop 'audit_elasticsearch_password' $PROPFILE)
audit_solr_urls=$(get_prop 'audit_solr_urls' $PROPFILE)
audit_solr_user=$(get_prop 'audit_solr_user' $PROPFILE)
audit_solr_password=$(get_prop 'audit_solr_password' $PROPFILE)
@@ -247,6 +251,16 @@ init_variables(){
exit 1
fi
fi
+ if [ "${audit_store}" == "elasticsearch" ] ;then
+ if [ "${audit_elasticsearch_urls}" == "" ] ;then
+ log "[I] Please provide valid URL for 'elasticsearch' audit store!"
+ exit 1
+ fi
+ if [ "${audit_elasticsearch_port}" == "" ] ;then
+ log "[I] Please provide valid port for 'elasticsearch' audit store!"
+ exit 1
+ fi
+ fi
db_ssl_enabled=`echo $db_ssl_enabled | tr '[:upper:]' '[:lower:]'`
if [ "${db_ssl_enabled}" != "true" ]
@@ -699,6 +713,25 @@ update_properties() {
updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
fi
+ if [ "${audit_store}" == "elasticsearch" ]
+ then
+ propertyName=ranger.audit.elasticsearch.urls
+ newPropertyValue=${audit_elasticsearch_urls}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+
+ propertyName=ranger.audit.elasticsearch.port
+ newPropertyValue=${audit_elasticsearch_port}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+
+ propertyName=ranger.audit.elasticsearch.user
+ newPropertyValue=${audit_elasticsearch_user}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+
+ propertyName=ranger.audit.elasticsearch.password
+ newPropertyValue=${audit_elasticsearch_password}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+ fi
+
if [ "${audit_store}" != "" ]
then
propertyName=ranger.audit.source.type
diff --git a/security-admin/scripts/upgrade_admin.py b/security-admin/scripts/upgrade_admin.py
index 28b7e98..903f8bb 100755
--- a/security-admin/scripts/upgrade_admin.py
+++ b/security-admin/scripts/upgrade_admin.py
@@ -105,6 +105,10 @@ config2xmlMAP = {
'xa.logs.base.dir':'ranger.logs.base.dir',
'xa.scheduler.enabled':'ranger.scheduler.enabled',
'xa.audit.store':'ranger.audit.source.type',
+ 'audit_elasticsearch_urls':'ranger.audit.elasticsearch.urls',
+ 'audit_elasticsearch_port':'ranger.audit.elasticsearch.port',
+ 'audit_elasticsearch_user':'ranger.audit.elasticsearch.user',
+ 'audit_elasticsearch_password':'ranger.audit.elasticsearch.password',
'audit_solr_urls':'ranger.audit.solr.urls',
'auditDB.jdbc.dialect':'ranger.jpa.audit.jdbc.dialect',
'auditDB.jdbc.driver':'ranger.jpa.audit.jdbc.driver',
diff --git a/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java
new file mode 100644
index 0000000..e902e65
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.SearchField;
+import org.apache.ranger.common.SortField;
+import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.*;
+
+public class AccessAuditsService {
+ protected List<SortField> sortFields = new ArrayList<SortField>();
+ protected List<SearchField> searchFields;
+ @Autowired
+ protected
+ RESTErrorUtil restErrorUtil;
+ @Autowired
+ protected
+ RangerDaoManager daoManager;
+
+ public AccessAuditsService() {
+ searchFields = new ArrayList<SearchField>();
+ searchFields.add(new SearchField("id", "id",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("accessType", "access",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("aclEnforcer", "enforcer",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("agentId", "agent",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("repoName", "repo",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("sessionId", "sess",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("requestUser", "reqUser",
+ SearchField.DATA_TYPE.STR_LIST, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("excludeUser", "exlUser",
+ SearchField.DATA_TYPE.STR_LIST, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("requestData", "reqData", SearchField.DATA_TYPE.STRING,
+ SearchField.SEARCH_TYPE.PARTIAL));
+ searchFields.add(new SearchField("resourcePath", "resource", SearchField.DATA_TYPE.STRING,
+ SearchField.SEARCH_TYPE.PARTIAL));
+ searchFields.add(new SearchField("clientIP", "cliIP",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+
+ searchFields.add(new SearchField("auditType", "logType",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("accessResult", "result",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ // searchFields.add(new SearchField("assetId", "obj.assetId",
+ // SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("policyId", "policy",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("repoType", "repoType",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("-repoType", "-repoType",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("-requestUser", "-reqUser",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("resourceType", "resType",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("reason", "reason",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("action", "action",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+
+ searchFields.add(new SearchField("startDate", "evtTime",
+ SearchField.DATA_TYPE.DATE, SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN));
+ searchFields.add(new SearchField("endDate", "evtTime", SearchField.DATA_TYPE.DATE,
+ SearchField.SEARCH_TYPE.LESS_EQUAL_THAN));
+
+ searchFields.add(new SearchField("tags", "tags", SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL));
+ searchFields.add(new SearchField("cluster", "cluster",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("zoneName", "zoneName",
+ SearchField.DATA_TYPE.STR_LIST, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("agentHost", "agentHost",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL));
+
+ sortFields.add(new SortField("eventTime", "evtTime", true,
+ SortField.SORT_ORDER.DESC));
+ }
+
+ protected void updateUserExclusion(Map<String, Object> paramList) {
+ String val = (String) paramList.get("excludeServiceUser");
+
+ if (val != null && Boolean.valueOf(val.trim())) {
+ // add param to negate requestUsers which will be added as filter query
+ List<String> excludeUsersList = getExcludeUsersList();
+ if (CollectionUtils.isNotEmpty(excludeUsersList)) {
+ Object oldUserExclusions = paramList.get("-requestUser");
+ if (oldUserExclusions instanceof Collection && (!((Collection<?>)oldUserExclusions).isEmpty())) {
+ excludeUsersList.addAll((Collection<String>)oldUserExclusions);
+ paramList.put("-requestUser", excludeUsersList);
+ } else {
+ paramList.put("-requestUser", excludeUsersList);
+ }
+ }
+ }
+ }
+
+ private List<String> getExcludeUsersList() {
+ //for excluding serviceUsers using existing property in ranger-admin-site
+ List<String> excludeUsersList = new ArrayList<String>(getServiceUserList());
+
+ //for excluding additional users using new property in ranger-admin-site
+ String additionalExcludeUsers = PropertiesUtil.getProperty("ranger.accesslogs.exclude.users.list");
+ List<String> additionalExcludeUsersList = null;
+ if (StringUtils.isNotBlank(additionalExcludeUsers)) {
+ additionalExcludeUsersList = new ArrayList<>(Arrays.asList(StringUtils.split(additionalExcludeUsers, ",")));
+ for (String serviceUser : additionalExcludeUsersList) {
+ if (StringUtils.isNotBlank(serviceUser) && !excludeUsersList.contains(serviceUser.trim())) {
+ excludeUsersList.add(serviceUser);
+ }
+ }
+ }
+ return excludeUsersList;
+ }
+
+ private List<String> getServiceUserList() {
+ String components = EmbeddedServiceDefsUtil.DEFAULT_BOOTSTRAP_SERVICEDEF_LIST;
+ List<String> serviceUsersList = new ArrayList<String>();
+ List<String> componentNames = Arrays.asList(StringUtils.split(components,","));
+ for(String componentName : componentNames) {
+ String serviceUser = PropertiesUtil.getProperty("ranger.plugins."+componentName+".serviceuser");
+ if(StringUtils.isNotBlank(serviceUser)) {
+ serviceUsersList.add(serviceUser);
+ }
+ }
+ return serviceUsersList;
+ }
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
index 17c105f..d3ce251 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
@@ -48,6 +48,7 @@ import org.apache.ranger.common.RangerConstants;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.StringUtil;
import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.elasticsearch.ElasticSearchAccessAuditsService;
import org.apache.ranger.entity.XXPermMap;
import org.apache.ranger.entity.XXPluginInfo;
import org.apache.ranger.entity.XXPolicyExportAudit;
@@ -110,6 +111,9 @@ public class AssetMgr extends AssetMgrBase {
SolrAccessAuditsService solrAccessAuditsService;
@Autowired
+ ElasticSearchAccessAuditsService elasticSearchAccessAuditsService;
+
+ @Autowired
XPolicyService xPolicyService;
@Autowired
@@ -1122,6 +1126,8 @@ public class AssetMgr extends AssetMgrBase {
if (RangerBizUtil.AUDIT_STORE_SOLR.equalsIgnoreCase(xaBizUtil.getAuditDBType())) {
return solrAccessAuditsService.searchXAccessAudits(searchCriteria);
+ } else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(xaBizUtil.getAuditDBType())) {
+ return elasticSearchAccessAuditsService.searchXAccessAudits(searchCriteria);
} else {
return xAccessAuditService.searchXAccessAudits(searchCriteria);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
index ebc72cf..4fb21a0 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
@@ -110,6 +110,7 @@ public class RangerBizUtil {
private static int PATH_CHAR_SET_LEN = PATH_CHAR_SET.length;
public static final String AUDIT_STORE_RDBMS = "DB";
public static final String AUDIT_STORE_SOLR = "solr";
+ public static final String AUDIT_STORE_ElasticSearch = "elasticSearch";
public static final boolean batchClearEnabled = PropertiesUtil.getBooleanProperty("ranger.jpa.jdbc.batch-clear.enable", true);
public static final int policyBatchSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10);
public static final int batchPersistSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-persist.size", 500);
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
index 9eecc20..4e5410e 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
@@ -24,6 +24,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.ranger.common.ContextUtil;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.UserSessionBase;
+import org.apache.ranger.elasticsearch.ElasticSearchAccessAuditsService;
import org.apache.ranger.solr.SolrAccessAuditsService;
import org.apache.ranger.view.VXAccessAudit;
import org.apache.ranger.view.VXAccessAuditList;
@@ -41,6 +42,9 @@ public class XAuditMgr extends XAuditMgrBase {
SolrAccessAuditsService solrAccessAuditsService;
@Autowired
+ ElasticSearchAccessAuditsService elasticSearchAccessAuditsService;
+
+ @Autowired
RangerBizUtil rangerBizUtil;
public VXTrxLog getXTrxLog(Long id) {
@@ -110,8 +114,11 @@ public class XAuditMgr extends XAuditMgrBase {
@Override
public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
- if ("solr".equalsIgnoreCase(rangerBizUtil.getAuditDBType())) {
+ String auditDBType = rangerBizUtil.getAuditDBType();
+ if (RangerBizUtil.AUDIT_STORE_SOLR.equalsIgnoreCase(auditDBType)) {
return solrAccessAuditsService.searchXAccessAudits(searchCriteria);
+ } else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(auditDBType)) {
+ return elasticSearchAccessAuditsService.searchXAccessAudits(searchCriteria);
} else {
return super.searchXAccessAudits(searchCriteria);
}
@@ -119,8 +126,11 @@ public class XAuditMgr extends XAuditMgrBase {
@Override
public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
- if ("solr".equalsIgnoreCase(rangerBizUtil.getAuditDBType())) {
+ String auditDBType = rangerBizUtil.getAuditDBType();
+ if (RangerBizUtil.AUDIT_STORE_SOLR.equalsIgnoreCase(auditDBType)) {
return solrAccessAuditsService.getXAccessAuditSearchCount(searchCriteria);
+ } else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(auditDBType)) {
+ return elasticSearchAccessAuditsService.getXAccessAuditSearchCount(searchCriteria);
} else {
return super.getXAccessAuditSearchCount(searchCriteria);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
new file mode 100644
index 0000000..c88726f
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.elasticsearch;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.common.MessageEnums;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.db.XXServiceDefDao;
+import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.view.VXAccessAudit;
+import org.apache.ranger.view.VXAccessAuditList;
+import org.apache.ranger.view.VXLong;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@Scope("singleton")
+public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAuditsService {
+ private static final Logger LOGGER = Logger.getLogger(ElasticSearchAccessAuditsService.class);
+
+ @Autowired
+ ElasticSearchMgr elasticSearchMgr;
+
+ @Autowired
+ ElasticSearchUtil elasticSearchUtil;
+
+
+ public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
+
+ RestHighLevelClient client = elasticSearchMgr.getClient();
+ final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true);
+ if (client == null) {
+ LOGGER.warn("ElasticSearch client is null, so not running the query.");
+ throw restErrorUtil.createRESTException(
+ "Error connecting to search engine",
+ MessageEnums.ERROR_SYSTEM);
+ }
+ List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
+ Map<String, Object> paramList = searchCriteria.getParamList();
+ updateUserExclusion(paramList);
+
+ SearchResponse response;
+ try {
+ response = elasticSearchUtil.searchResources(searchCriteria, searchFields, sortFields, client, elasticSearchMgr.index);
+ } catch (IOException e) {
+ LOGGER.warn(String.format("ElasticSearch query failed: %s", e.getMessage()));
+ throw restErrorUtil.createRESTException(
+ "Error querying search engine",
+ MessageEnums.ERROR_SYSTEM);
+ }
+ MultiGetItemResponse[] docs;
+ try {
+ docs = elasticSearchUtil.fetch(client, elasticSearchMgr.index, response.getHits().getHits());
+ } catch (IOException e) {
+ LOGGER.warn(String.format("ElasticSearch fetch failed: %s", e.getMessage()));
+ throw restErrorUtil.createRESTException(
+ "Error querying search engine",
+ MessageEnums.ERROR_SYSTEM);
+ }
+ for (int i = 0; i < docs.length; i++) { // NOPMD - This for loop can be replaced by a foreach loop
+ MultiGetItemResponse doc = docs[i];
+ VXAccessAudit vXAccessAudit = populateViewBean(doc.getResponse());
+ if (vXAccessAudit != null) {
+ String serviceType = vXAccessAudit.getServiceType();
+ boolean isHive = "hive".equalsIgnoreCase(serviceType);
+ if (!hiveQueryVisibility && isHive) {
+ vXAccessAudit.setRequestData(null);
+ } else if (isHive) {
+ String accessType = vXAccessAudit.getAccessType();
+ if ("grant".equalsIgnoreCase(accessType)
+ || "revoke".equalsIgnoreCase(accessType)) {
+ String requestData = vXAccessAudit.getRequestData();
+ if (requestData != null) {
+ try {
+ vXAccessAudit.setRequestData(
+ java.net.URLDecoder.decode(requestData, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.warn("Error while encoding request data: " + requestData, e);
+ }
+ } else {
+ LOGGER.warn(
+ "Error in request data of audit from elasticSearch. AuditData: "
+ + vXAccessAudit.toString());
+ }
+ }
+ }
+ }
+ xAccessAuditList.add(vXAccessAudit);
+ }
+
+ VXAccessAuditList returnList = new VXAccessAuditList();
+ returnList.setPageSize(searchCriteria.getMaxRows());
+ returnList.setResultSize(response.getHits().getHits().length);
+ returnList.setTotalCount(response.getHits().getTotalHits());
+ returnList.setStartIndex(searchCriteria.getStartIndex());
+ returnList.setVXAccessAudits(xAccessAuditList);
+ return returnList;
+ }
+
+ public void setRestErrorUtil(RESTErrorUtil restErrorUtil) {
+ this.restErrorUtil = restErrorUtil;
+ }
+
+
+ /**
+ * @param doc
+ * @return
+ */
+ private VXAccessAudit populateViewBean(GetResponse doc) {
+ VXAccessAudit accessAudit = new VXAccessAudit();
+
+ Object value = null;
+ if(LOGGER.isDebugEnabled()) {
+ LOGGER.debug("doc=" + doc.toString());
+ }
+
+ Map<String, Object> source = doc.getSource();
+ value = source.get("id");
+ if (value != null) {
+ // TODO: Converting ID to hashcode for now
+ accessAudit.setId((long) value.hashCode());
+ }
+
+ value = source.get("cluster");
+ if (value != null) {
+ accessAudit.setClusterName(value.toString());
+ }
+
+ value = source.get("zoneName");
+ if (value != null) {
+ accessAudit.setZoneName(value.toString());
+ }
+
+ value = source.get("agentHost");
+ if (value != null) {
+ accessAudit.setAgentHost(value.toString());
+ }
+
+ value = source.get("policyVersion");
+ if (value != null) {
+ accessAudit.setPolicyVersion(elasticSearchUtil.toLong(value));
+ }
+
+ value = source.get("access");
+ if (value != null) {
+ accessAudit.setAccessType(value.toString());
+ }
+
+ value = source.get("enforcer");
+ if (value != null) {
+ accessAudit.setAclEnforcer(value.toString());
+ }
+ value = source.get("agent");
+ if (value != null) {
+ accessAudit.setAgentId(value.toString());
+ }
+ value = source.get("repo");
+ if (value != null) {
+ accessAudit.setRepoName(value.toString());
+ }
+ value = source.get("sess");
+ if (value != null) {
+ accessAudit.setSessionId(value.toString());
+ }
+ value = source.get("reqUser");
+ if (value != null) {
+ accessAudit.setRequestUser(value.toString());
+ }
+ value = source.get("reqData");
+ if (value != null) {
+ accessAudit.setRequestData(value.toString());
+ }
+ value = source.get("resource");
+ if (value != null) {
+ accessAudit.setResourcePath(value.toString());
+ }
+ value = source.get("cliIP");
+ if (value != null) {
+ accessAudit.setClientIP(value.toString());
+ }
+ value = source.get("logType");
+ //if (value != null) {
+ // TODO: Need to see what logType maps to in UI
+// accessAudit.setAuditType(solrUtil.toInt(value));
+ //}
+ value = source.get("result");
+ if (value != null) {
+ accessAudit.setAccessResult(elasticSearchUtil.toInt(value));
+ }
+ value = source.get("policy");
+ if (value != null) {
+ accessAudit.setPolicyId(elasticSearchUtil.toLong(value));
+ }
+ value = source.get("repoType");
+ if (value != null) {
+ accessAudit.setRepoType(elasticSearchUtil.toInt(value));
+ if(null != daoManager) {
+ XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef();
+ if(xxServiceDef != null) {
+ XXServiceDef xServiceDef = xxServiceDef.getById((long) accessAudit.getRepoType());
+ if (xServiceDef != null) {
+ accessAudit.setServiceType(xServiceDef.getName());
+ }
+ }
+ }
+ }
+ value = source.get("resType");
+ if (value != null) {
+ accessAudit.setResourceType(value.toString());
+ }
+ value = source.get("reason");
+ if (value != null) {
+ accessAudit.setResultReason(value.toString());
+ }
+ value = source.get("action");
+ if (value != null) {
+ accessAudit.setAction(value.toString());
+ }
+ value = source.get("evtTime");
+ if (value != null) {
+ accessAudit.setEventTime(elasticSearchUtil.toDate(value));
+ }
+ value = source.get("seq_num");
+ if (value != null) {
+ accessAudit.setSequenceNumber(elasticSearchUtil.toLong(value));
+ }
+ value = source.get("event_count");
+ if (value != null) {
+ accessAudit.setEventCount(elasticSearchUtil.toLong(value));
+ }
+ value = source.get("event_dur_ms");
+ if (value != null) {
+ accessAudit.setEventDuration(elasticSearchUtil.toLong(value));
+ }
+ value = source.get("tags");
+ if (value != null) {
+ accessAudit.setTags(value.toString());
+ }
+ return accessAudit;
+ }
+
+ /**
+ * @param searchCriteria
+ * @return
+ */
+ public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
+ long count = 100;
+ VXLong vXLong = new VXLong();
+ vXLong.setValue(count);
+ return vXLong;
+ }
+
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java
new file mode 100644
index 0000000..a060877
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchMgr.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.elasticsearch;
+
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.log4j.Logger;
+import org.apache.ranger.audit.destination.ElasticSearchAuditDestination;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.StringUtil;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.springframework.stereotype.Component;
+
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.*;
+
+/**
+ * This class initializes the ElasticSearch client
+ *
+ */
+@Component
+public class ElasticSearchMgr {
+
+ private static final Logger logger = Logger.getLogger(ElasticSearchMgr.class);
+ public String index;
+
+ synchronized void connect() {
+ if (client == null) {
+ synchronized (ElasticSearchAuditDestination.class) {
+ if (client == null) {
+
+ String urls = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_URLS);
+ String protocol = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PROTOCOL, "http");
+ String user = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_USER, "");
+ String password = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PWRD, "");
+ int port = Integer.parseInt(PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PORT));
+ this.index = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_INDEX, "audit");
+ String parameterString = String.format("(URLs: %s; Protocol: %s; Port: %s; User: %s)",
+ urls, protocol, port, user);
+ logger.info("Initializing ElasticSearch " + parameterString);
+ if (urls != null) {
+ urls = urls.trim();
+ }
+ if (!new StringUtil().isEmpty(urls) && urls.equalsIgnoreCase("NONE")) {
+ logger.info(String.format("Clearing URI config value: %s", urls));
+ urls = null;
+ }
+
+ try {
+
+ final CredentialsProvider credentialsProvider;
+ if(!user.isEmpty()) {
+ credentialsProvider = new BasicCredentialsProvider();
+ logger.info(String.format("Using %s to login to ElasticSearch", user));
+ credentialsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(user, password));
+ } else {
+ credentialsProvider = null;
+ }
+ client = new RestHighLevelClient(
+ RestClient.builder(
+ MiscUtil.toArray(urls, ",").stream()
+ .map(x -> new HttpHost(x, port, protocol))
+ .<HttpHost>toArray(i -> new HttpHost[i])
+ ).setHttpClientConfigCallback(clientBuilder ->
+ (credentialsProvider != null) ? clientBuilder.setDefaultCredentialsProvider(credentialsProvider) : clientBuilder));
+ } catch (Throwable t) {
+ logger.fatal("Can't connect to ElasticSearch: " + parameterString, t);
+ }
+ }
+ }
+ }
+ }
+
+ RestHighLevelClient client = null;
+ public RestHighLevelClient getClient() {
+ if(client !=null) {
+ return client;
+ } else {
+ connect();
+ }
+ return client;
+ }
+
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
new file mode 100644
index 0000000..32a4583
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.elasticsearch;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.common.*;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.sort.SortOrder;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+
+@Component
+public class ElasticSearchUtil {
+ private static final Logger logger = Logger.getLogger(ElasticSearchUtil.class);
+
+ @Autowired
+ StringUtil stringUtil;
+
+ String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+ SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr);
+
+ public ElasticSearchUtil() {
+ String timeZone = PropertiesUtil.getProperty("xa.elasticSearch.timezone");
+ if (timeZone != null) {
+ logger.info("Setting timezone to " + timeZone);
+ try {
+ dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
+ } catch (Throwable t) {
+ logger.error("Error setting timezone. TimeZone = " + timeZone);
+ }
+ }
+ }
+
+
+ // Utility methods
+ public int toInt(Object value) {
+ if (value == null) {
+ return 0;
+ }
+ if (value instanceof Integer) {
+ return (Integer) value;
+ }
+ if (value.toString().isEmpty()) {
+ return 0;
+ }
+ try {
+ return Integer.valueOf(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to integer. Value = " + value, t);
+ }
+ return 0;
+ }
+
+ public long toLong(Object value) {
+ if (value == null) {
+ return 0;
+ }
+ if (value instanceof Long) {
+ return (Long) value;
+ }
+ if (value.toString().isEmpty()) {
+ return 0;
+ }
+ try {
+ return Long.valueOf(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to long. Value = " + value, t);
+ }
+ return 0;
+ }
+
+ public Date toDate(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date) value;
+ }
+ try {
+ LocalDateTime localDateTime = LocalDateTime.parse(value.toString(), DateTimeFormatter.ISO_DATE_TIME);
+ return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant());
+ } catch (Throwable t) {
+ logger.error("Error converting value to date. Value = " + value, t);
+ }
+ return null;
+ }
+
+ public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException {
+ // See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html
+ QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria);
+ if (searchCriteria.getParamList() != null) {
+ searchFields.stream().forEach(queryAccumulator::addQuery);
+ // For now assuming there is only date field where range query will
+ // be done. If we there are more than one, then we should create a
+ // hashmap for each field name
+ if (queryAccumulator.fromDate != null || queryAccumulator.toDate != null) {
+ queryAccumulator.queries.add(setDateRange(queryAccumulator.dateFieldName, queryAccumulator.fromDate, queryAccumulator.toDate));
+ }
+ }
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ queryAccumulator.queries.stream().filter(x -> x != null).forEach(boolQueryBuilder::must);
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ setSortClause(searchCriteria, sortFields, searchSourceBuilder);
+ searchSourceBuilder.from(searchCriteria.getStartIndex());
+ searchSourceBuilder.size(searchCriteria.getMaxRows());
+ searchSourceBuilder.fetchSource(true);
+ SearchRequest query = new SearchRequest();
+ query.indices(index);
+ query.source(searchSourceBuilder.query(boolQueryBuilder));
+ return client.search(query);
+ }
+
+ public void setSortClause(SearchCriteria searchCriteria,
+ List<SortField> sortFields,
+ SearchSourceBuilder searchSourceBuilder) {
+
+ // TODO: We are supporting single sort field only for now
+ String sortBy = searchCriteria.getSortBy();
+ String querySortBy = null;
+ if (!stringUtil.isEmpty(sortBy)) {
+ sortBy = sortBy.trim();
+ for (SortField sortField : sortFields) {
+ if (sortBy.equalsIgnoreCase(sortField.getParamName())) {
+ querySortBy = sortField.getFieldName();
+ // Override the sortBy using the normalized value
+ searchCriteria.setSortBy(sortField.getParamName());
+ break;
+ }
+ }
+ }
+
+ if (querySortBy == null) {
+ for (SortField sortField : sortFields) {
+ if (sortField.isDefault()) {
+ querySortBy = sortField.getFieldName();
+ // Override the sortBy using the default value
+ searchCriteria.setSortBy(sortField.getParamName());
+ searchCriteria.setSortType(sortField.getDefaultOrder().name());
+ break;
+ }
+ }
+ }
+
+ if (querySortBy != null) {
+ // Add sort type
+ String sortType = searchCriteria.getSortType();
+ SortOrder order = SortOrder.ASC;
+ if (sortType != null && "desc".equalsIgnoreCase(sortType)) {
+ order = SortOrder.DESC;
+ }
+ searchSourceBuilder.sort(querySortBy, order);
+ }
+ }
+
+ public QueryBuilder orList(String fieldName, Collection<?> valueList) {
+ if (valueList == null || valueList.isEmpty()) {
+ return null;
+ }
+ if (valueList.isEmpty()) {
+ return null;
+ } else {
+ return QueryBuilders.queryStringQuery(valueList.stream()
+ .map(this::filterText)
+ .map(x -> "(" + x + ")")
+ .reduce((a, b) -> a + " OR " + b)
+ .get()
+ ).defaultField(fieldName);
+ }
+ }
+
+
+ private String filterText(Object value) {
+ return ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase());
+ }
+
+ public QueryBuilder setDateRange(String fieldName, Date fromDate, Date toDate) {
+ RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(fieldName).format(dateFormateStr);
+ if (fromDate != null) {
+ rangeQueryBuilder.from(dateFormat.format(fromDate));
+ }
+ if (toDate != null) {
+ rangeQueryBuilder.to(dateFormat.format(toDate));
+ }
+ return rangeQueryBuilder;
+ }
+
+ public MultiGetItemResponse[] fetch(RestHighLevelClient client, String index, SearchHit... hits) throws IOException {
+ if(0 == hits.length) {
+ return new MultiGetItemResponse[0];
+ }
+ MultiGetRequest multiGetRequest = new MultiGetRequest();
+ for (SearchHit hit : hits) {
+ MultiGetRequest.Item item = new MultiGetRequest.Item(index, null, hit.getId());
+ item.fetchSourceContext(FetchSourceContext.FETCH_SOURCE);
+ multiGetRequest.add(item);
+ }
+ return client.multiGet(multiGetRequest).getResponses();
+ }
+
+ private class QueryAccumulator {
+ public final List<QueryBuilder> queries = new ArrayList<>();
+ public final SearchCriteria searchCriteria;
+ public Date fromDate;
+ public Date toDate;
+ public String dateFieldName;
+
+ private QueryAccumulator(SearchCriteria searchCriteria) {
+ this.searchCriteria = searchCriteria;
+ this.fromDate = null;
+ this.toDate = null;
+ this.dateFieldName = null;
+ }
+
+ public QueryAccumulator addQuery(SearchField searchField) {
+ QueryBuilder queryBuilder = getQueryBuilder(searchField);
+ if (null != queryBuilder) {
+ queries.add(queryBuilder);
+ }
+ return this;
+ }
+
+ public QueryBuilder getQueryBuilder(SearchField searchField) {
+ String clientFieldName = searchField.getClientFieldName();
+ String fieldName = searchField.getFieldName();
+ SearchField.DATA_TYPE dataType = searchField.getDataType();
+ SearchField.SEARCH_TYPE searchType = searchField.getSearchType();
+ Object paramValue = searchCriteria.getParamValue(clientFieldName);
+ return getQueryBuilder(dataType, searchType, fieldName, paramValue);
+ }
+
+ private QueryBuilder getQueryBuilder(SearchField.DATA_TYPE dataType, SearchField.SEARCH_TYPE searchType, String fieldName, Object paramValue) {
+ if (paramValue == null || paramValue.toString().isEmpty()) {
+ return null;
+ }
+ if (fieldName.startsWith("-")) {
+ QueryBuilder negativeQuery = getQueryBuilder(dataType, searchType, fieldName.substring(1), paramValue);
+ return null == negativeQuery ? null : QueryBuilders.boolQuery().mustNot(negativeQuery);
+ }
+ if (paramValue instanceof Collection) {
+ Collection<?> valueList = (Collection<?>) paramValue;
+ if (valueList.isEmpty()) {
+ return null;
+ } else {
+ return QueryBuilders.queryStringQuery(valueList.stream()
+ .map(ElasticSearchUtil.this::filterText)
+ .map(x -> "(" + x + ")")
+ .reduce((a, b) -> a + " OR " + b)
+ .get()
+ ).defaultField(fieldName);
+ }
+ } else {
+ if (dataType == SearchField.DATA_TYPE.DATE) {
+ if (!(paramValue instanceof Date)) {
+ logger.error(String.format(
+ "Search value is not a Java Date Object: %s %s %s",
+ fieldName, searchType, paramValue));
+ } else {
+ if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN
+ || searchType == SearchField.SEARCH_TYPE.GREATER_THAN) {
+ fromDate = (Date) paramValue;
+ dateFieldName = fieldName;
+ } else if (searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN
+ || searchType == SearchField.SEARCH_TYPE.LESS_THAN) {
+ toDate = (Date) paramValue;
+ dateFieldName = fieldName;
+ }
+ }
+ return null;
+ } else if (searchType == SearchField.SEARCH_TYPE.GREATER_EQUAL_THAN
+ || searchType == SearchField.SEARCH_TYPE.GREATER_THAN
+ || searchType == SearchField.SEARCH_TYPE.LESS_EQUAL_THAN
+ || searchType == SearchField.SEARCH_TYPE.LESS_THAN) { //NOPMD
+ logger.warn(String.format("Range Queries Not Implemented: %s %s %s",
+ fieldName, searchType, paramValue));
+ return null;
+ } else {
+ if (searchType == SearchField.SEARCH_TYPE.PARTIAL) {
+ if (paramValue.toString().trim().length() == 0) {
+ return null;
+ } else {
+ return QueryBuilders.queryStringQuery("*" + filterText(paramValue) + "*").defaultField(fieldName);
+ }
+ } else {
+ if (paramValue.toString().trim().length() > 0) {
+ return QueryBuilders.matchPhraseQuery(fieldName, filterText(paramValue));
+ } else {
+ return null;
+ }
+ }
+ }
+ }
+ }
+
+ }
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
index 453c7dc..f3c6e3f 100644
--- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
@@ -21,28 +21,17 @@ package org.apache.ranger.solr;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
+import org.apache.ranger.AccessAuditsService;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.common.RESTErrorUtil;
import org.apache.ranger.common.SearchCriteria;
-import org.apache.ranger.common.SearchField;
-import org.apache.ranger.common.SearchField.DATA_TYPE;
-import org.apache.ranger.common.SearchField.SEARCH_TYPE;
-import org.apache.ranger.common.SortField;
-import org.apache.ranger.common.SortField.SORT_ORDER;
-import org.apache.ranger.common.StringUtil;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.XXService;
import org.apache.ranger.entity.XXServiceDef;
-import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.view.VXAccessAudit;
import org.apache.ranger.view.VXAccessAuditList;
import org.apache.ranger.view.VXLong;
@@ -56,7 +45,7 @@ import org.springframework.stereotype.Service;
@Service
@Scope("singleton")
-public class SolrAccessAuditsService {
+public class SolrAccessAuditsService extends AccessAuditsService {
private static final Logger LOGGER = Logger.getLogger(SolrAccessAuditsService.class);
@Autowired
@@ -69,77 +58,8 @@ public class SolrAccessAuditsService {
RESTErrorUtil restErrorUtil;
@Autowired
- StringUtil stringUtil;
-
- @Autowired
RangerDaoManager daoManager;
- private List<SortField> sortFields = new ArrayList<SortField>();
- private List<SearchField> searchFields = new ArrayList<SearchField>();
-
- public SolrAccessAuditsService() {
-
- searchFields.add(new SearchField("id", "id",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("accessType", "access",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("aclEnforcer", "enforcer",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("agentId", "agent",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("repoName", "repo",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("sessionId", "sess",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("requestUser", "reqUser",
- SearchField.DATA_TYPE.STR_LIST, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("excludeUser", "exlUser",
- SearchField.DATA_TYPE.STR_LIST, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("requestData", "reqData", SearchField.DATA_TYPE.STRING,
- SearchField.SEARCH_TYPE.PARTIAL));
- searchFields.add(new SearchField("resourcePath", "resource", SearchField.DATA_TYPE.STRING,
- SearchField.SEARCH_TYPE.PARTIAL));
- searchFields.add(new SearchField("clientIP", "cliIP",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
-
- searchFields.add(new SearchField("auditType", "logType",
- SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("accessResult", "result",
- SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
- // searchFields.add(new SearchField("assetId", "obj.assetId",
- // SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("policyId", "policy",
- SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("repoType", "repoType",
- SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("-repoType", "-repoType",
- SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("-requestUser", "-reqUser",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("resourceType", "resType",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("reason", "reason",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("action", "action",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
-
- searchFields.add(new SearchField("startDate", "evtTime",
- DATA_TYPE.DATE, SEARCH_TYPE.GREATER_EQUAL_THAN));
- searchFields.add(new SearchField("endDate", "evtTime", DATA_TYPE.DATE,
- SEARCH_TYPE.LESS_EQUAL_THAN));
-
- searchFields.add(new SearchField("tags", "tags", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL));
- searchFields.add(new SearchField("cluster", "cluster",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("zoneName", "zoneName",
- SearchField.DATA_TYPE.STR_LIST, SearchField.SEARCH_TYPE.FULL));
- searchFields.add(new SearchField("agentHost", "agentHost",
- SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL));
-
- sortFields.add(new SortField("eventTime", "evtTime", true,
- SORT_ORDER.DESC));
- }
-
public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
@@ -191,56 +111,6 @@ public class SolrAccessAuditsService {
return returnList;
}
-
- private void updateUserExclusion(Map<String, Object> paramList) {
- String val = (String) paramList.get("excludeServiceUser");
-
- if (val != null && Boolean.valueOf(val.trim())) { // add param to negate requestUsers which will be added as
- // filter query in solr
- List<String> excludeUsersList = getExcludeUsersList();
- if (CollectionUtils.isNotEmpty(excludeUsersList)) {
- Object oldUserExclusions = paramList.get("-requestUser");
- if (oldUserExclusions instanceof Collection && (!((Collection<?>)oldUserExclusions).isEmpty())) {
- excludeUsersList.addAll((Collection<String>)oldUserExclusions);
- paramList.put("-requestUser", excludeUsersList);
- } else {
- paramList.put("-requestUser", excludeUsersList);
- }
- }
- }
- }
-
- private List<String> getExcludeUsersList() {
- //for excluding serviceUsers using existing property in ranger-admin-site
- List<String> excludeUsersList = new ArrayList<String>(getServiceUserList());
-
- //for excluding additional users using new property in ranger-admin-site
- String additionalExcludeUsers = PropertiesUtil.getProperty("ranger.accesslogs.exclude.users.list");
- List<String> additionalExcludeUsersList = null;
- if (StringUtils.isNotBlank(additionalExcludeUsers)) {
- additionalExcludeUsersList = new ArrayList<>(Arrays.asList(StringUtils.split(additionalExcludeUsers, ",")));
- for (String serviceUser : additionalExcludeUsersList) {
- if (StringUtils.isNotBlank(serviceUser) && !excludeUsersList.contains(serviceUser.trim())) {
- excludeUsersList.add(serviceUser);
- }
- }
- }
- return excludeUsersList;
- }
-
- private List<String> getServiceUserList() {
- String components = EmbeddedServiceDefsUtil.DEFAULT_BOOTSTRAP_SERVICEDEF_LIST;
- List<String> serviceUsersList = new ArrayList<String>();
- List<String> componentNames = Arrays.asList(StringUtils.split(components,","));
- for(String componentName : componentNames) {
- String serviceUser = PropertiesUtil.getProperty("ranger.plugins."+componentName+".serviceuser");
- if(StringUtils.isNotBlank(serviceUser)) {
- serviceUsersList.add(serviceUser);
- }
- }
- return serviceUsersList;
- }
-
/**
* @param doc
* @return
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
index 298f02b..5dc14ab 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
@@ -46,6 +46,26 @@
<description></description>
</property>
<property>
+ <name>ranger.audit.elasticsearch.urls</name>
+ <value>127.0.0.1</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.audit.elasticsearch.port</name>
+ <value>9200</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.audit.elasticsearch.user</name>
+ <value></value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.audit.elasticsearch.password</name>
+ <value></value>
+ <description></description>
+ </property>
+ <property>
<name>ranger.audit.solr.urls</name>
<value>http://##solr_host##:6083/solr/ranger_audits</value>
<description></description>
diff --git a/security-admin/src/test/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsServiceTest.java b/security-admin/src/test/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsServiceTest.java
new file mode 100644
index 0000000..cebe5f5
--- /dev/null
+++ b/security-admin/src/test/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsServiceTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.elasticsearch;
+
+import static org.apache.ranger.audit.destination.ElasticSearchAuditDestination.CONFIG_PREFIX;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.log4j.Logger;
+import org.apache.ranger.audit.destination.ElasticSearchAuditDestination;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.StringUtil;
+import org.apache.ranger.view.VXAccessAuditList;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class ElasticSearchAccessAuditsServiceTest {
+ private static final Logger LOGGER = Logger.getLogger(ElasticSearchAccessAuditsServiceTest.class);
+
+ @Test
+ @Ignore // For manual execution only
+ public void testQuery() {
+ ElasticSearchAccessAuditsService elasticSearchAccessAuditsService = new ElasticSearchAccessAuditsService();
+ Map<String, String> properties = PropertiesUtil.getPropertiesMap();
+ properties.put("ranger.audit.elasticsearch.urls", "localhost");
+ properties.put("ranger.audit.elasticsearch.user", "elastic");
+ properties.put("ranger.audit.elasticsearch.password", "password1");
+ properties.put("ranger.audit.elasticsearch.port", "9200");
+ elasticSearchAccessAuditsService.elasticSearchMgr = new ElasticSearchMgr();
+ elasticSearchAccessAuditsService.elasticSearchUtil = new ElasticSearchUtil();
+ elasticSearchAccessAuditsService.elasticSearchUtil.stringUtil = new StringUtil();
+ elasticSearchAccessAuditsService.setRestErrorUtil(new RESTErrorUtil());
+ LOGGER.info("Running searchXAccessAudits:");
+ VXAccessAuditList vxAccessAuditList = elasticSearchAccessAuditsService.searchXAccessAudits(getSearchCriteria());
+ LOGGER.info(String.format("searchXAccessAudits results (%d items):", vxAccessAuditList.getListSize()));
+ ObjectWriter writer = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT).writer();
+ vxAccessAuditList.getVXAccessAudits().forEach(x -> {
+ try {
+ LOGGER.warn(writer.writeValueAsString(x));
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ });
+ }
+
+ private SearchCriteria getSearchCriteria() {
+ SearchCriteria searchCriteria = new SearchCriteria();
+ searchCriteria.setDistinct(false);
+ searchCriteria.setGetChildren(false);
+ searchCriteria.setGetCount(true);
+ searchCriteria.setMaxRows(25);
+ searchCriteria.setOwnerId(null);
+ searchCriteria.setSortBy("eventTime");
+ searchCriteria.setSortType("desc");
+ searchCriteria.setStartIndex(0);
+ Calendar calendar = Calendar.getInstance();
+ calendar.set(2019, 11,13);
+ searchCriteria.getParamList().put("startDate", calendar.getTime());
+ searchCriteria.getParamList().put("-repoType", 7);
+ searchCriteria.getParamList().put("-requestUser", new ArrayList<>());
+ searchCriteria.getParamList().put("requestUser", new ArrayList<>());
+ searchCriteria.getParamList().put("zoneName", new ArrayList<>());
+ return searchCriteria;
+ }
+
+ @Test
+ @Ignore // For manual execution only
+ public void testWrite() {
+ ElasticSearchAuditDestination elasticSearchAuditDestination = new ElasticSearchAuditDestination();
+ Properties properties = new Properties();
+ properties.put(CONFIG_PREFIX + "." + ElasticSearchAuditDestination.CONFIG_URLS, "localhost");
+ properties.put(CONFIG_PREFIX + "." + ElasticSearchAuditDestination.CONFIG_USER, "elastic");
+ properties.put(CONFIG_PREFIX + "." + ElasticSearchAuditDestination.CONFIG_PWRD, "password1");
+ elasticSearchAuditDestination.init(properties, CONFIG_PREFIX);
+ assert elasticSearchAuditDestination.log(Arrays.asList(getAuthzAuditEvent()));
+ }
+
+ private AuthzAuditEvent getAuthzAuditEvent() {
+ AuthzAuditEvent event = new AuthzAuditEvent();
+ event.setAccessResult((short) 1);
+ event.setAccessType("");
+ event.setAclEnforcer("");
+ event.setAction("");
+ event.setAdditionalInfo("");
+ event.setAgentHostname("");
+ event.setAgentId("");
+ event.setClientIP("");
+ event.setClusterName("");
+ event.setClientType("");
+ event.setEventCount(1);
+ event.setEventDurationMS(1);
+ event.setEventId("");
+ event.setEventTime(new Date());
+ event.setLogType("");
+ event.setPolicyId(1);
+ event.setPolicyVersion(1l);
+ event.setRepositoryName("");
+ event.setRequestData("");
+ event.setRepositoryName("");
+ event.setRepositoryType(1);
+ event.setResourcePath("");
+ event.setResultReason("");
+ event.setSeqNum(1);
+ event.setSessionId("");
+ event.setTags(new HashSet<>());
+ event.setUser("");
+ event.setZoneName("");
+ return event;
+ }
+}
diff --git a/storm-agent/conf/ranger-storm-audit-changes.cfg b/storm-agent/conf/ranger-storm-audit-changes.cfg
index 8071e7b..39d7955 100644
--- a/storm-agent/conf/ranger-storm-audit-changes.cfg
+++ b/storm-agent/conf/ranger-storm-audit-changes.cfg
@@ -52,6 +52,14 @@ xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-n
xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists
xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.elasticsearch.is.enabled %XAAUDIT.ELASTICSEARCH.IS_ENABLED% mod create-if-not-exists
+
+xasecure.audit.destination.elasticsearch %XAAUDIT.ELASTICSEARCH.ENABLE% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.urls %XAAUDIT.ELASTICSEARCH.URL% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.user %XAAUDIT.ELASTICSEARCH.USER% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.password %XAAUDIT.ELASTICSEARCH.PASSWORD% mod create-if-not-exists
+xasecure.audit.destination.elasticsearch.index %XAAUDIT.ELASTICSEARCH.INDEX% mod create-if-not-exists
+
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists
diff --git a/storm-agent/conf/ranger-storm-audit.xml b/storm-agent/conf/ranger-storm-audit.xml
index c0c66d4..77c71d6 100644
--- a/storm-agent/conf/ranger-storm-audit.xml
+++ b/storm-agent/conf/ranger-storm-audit.xml
@@ -250,7 +250,11 @@
<property>
<name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
- </property>
+ </property>
+ <property>
+ <name>xasecure.audit.elasticsearch.is.enabled</name>
+ <value>false</value>
+ </property>
<property>
<name>xasecure.audit.solr.async.max.queue.size</name>
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index e805b75..9fcd959 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -50,6 +50,18 @@ XAAUDIT.SOLR.PASSWORD=NONE
XAAUDIT.SOLR.ZOOKEEPER=NONE
XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/storm/audit/solr/spool
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+
# Enable audit logs to HDFS
#Example
#XAAUDIT.HDFS.ENABLE=true