You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2021/12/26 00:24:09 UTC
[ranger] branch master updated: RANGER-3540: Add support to read audit logs from Amazon CloudWatch
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 71744d3 RANGER-3540: Add support to read audit logs from Amazon CloudWatch
71744d3 is described below
commit 71744d3ff8e0db7f0a49881a891e77edf0ea518d
Author: pradeep <pr...@apache.org>
AuthorDate: Tue Dec 21 17:32:17 2021 +0530
RANGER-3540: Add support to read audit logs from Amazon CloudWatch
---
agents-audit/pom.xml | 2 +-
.../AmazonCloudWatchAuditDestination.java | 38 ++-
.../org/apache/ranger/audit/provider/MiscUtil.java | 72 ++++++
hbase-agent/conf/ranger-hbase-audit-changes.cfg | 1 +
hbase-agent/scripts/install.properties | 1 +
hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 1 +
hdfs-agent/scripts/install.properties | 1 +
hive-agent/conf/ranger-hive-audit-changes.cfg | 1 +
hive-agent/scripts/install.properties | 1 +
kms/scripts/install.properties | 1 +
knox-agent/conf/ranger-knox-audit-changes.cfg | 1 +
knox-agent/scripts/install.properties | 1 +
plugin-atlas/conf/ranger-atlas-audit-changes.cfg | 1 +
plugin-atlas/scripts/install.properties | 1 +
.../conf/ranger-elasticsearch-audit-changes.cfg | 1 +
plugin-elasticsearch/scripts/install.properties | 1 +
plugin-kafka/conf/ranger-kafka-audit-changes.cfg | 1 +
plugin-kafka/scripts/install.properties | 1 +
plugin-kms/conf/ranger-kms-audit-changes.cfg | 1 +
plugin-kylin/conf/ranger-kylin-audit-changes.cfg | 1 +
plugin-kylin/scripts/install.properties | 1 +
plugin-ozone/conf/ranger-ozone-audit-changes.cfg | 1 +
plugin-ozone/scripts/install.properties | 1 +
plugin-presto/conf/ranger-presto-audit-changes.cfg | 1 +
plugin-presto/scripts/install.properties | 1 +
plugin-solr/conf/ranger-solr-audit-changes.cfg | 1 +
plugin-solr/scripts/install.properties | 1 +
plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg | 1 +
plugin-sqoop/scripts/install.properties | 1 +
plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 1 +
plugin-yarn/scripts/install.properties | 1 +
pom.xml | 1 +
security-admin/pom.xml | 15 ++
security-admin/scripts/install.properties | 7 +-
.../scripts/ranger-admin-site-template.xml | 12 +
security-admin/scripts/setup.sh | 29 +++
security-admin/scripts/upgrade_admin.py | 3 +
.../org/apache/ranger/AccessAuditsService.java | 4 +
.../cloudwatch/CloudWatchAccessAuditsService.java | 288 +++++++++++++++++++++
.../ranger/amazon/cloudwatch/CloudWatchMgr.java | 77 ++++++
.../ranger/amazon/cloudwatch/CloudWatchUtil.java | 258 ++++++++++++++++++
.../main/java/org/apache/ranger/biz/AssetMgr.java | 6 +
.../java/org/apache/ranger/biz/RangerBizUtil.java | 1 +
.../main/java/org/apache/ranger/biz/XAuditMgr.java | 8 +
.../ElasticSearchAccessAuditsService.java | 17 +-
.../ranger/elasticsearch/ElasticSearchUtil.java | 54 ----
.../ranger/solr/SolrAccessAuditsService.java | 17 +-
.../main/java/org/apache/ranger/solr/SolrUtil.java | 52 ----
.../main/resources/conf.dist/ranger-admin-site.xml | 12 +
storm-agent/conf/ranger-storm-audit-changes.cfg | 1 +
storm-agent/scripts/install.properties | 1 +
51 files changed, 868 insertions(+), 135 deletions(-)
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index 5d031cc..1519249 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -36,7 +36,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
- <version>1.11.327</version>
+ <version>${com.amazonaws.aws-java-sdk-bom.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java
index b236a26..b3c7af4 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java
@@ -31,6 +31,9 @@ import com.amazonaws.services.logs.model.InputLogEvent;
import com.amazonaws.services.logs.model.InvalidSequenceTokenException;
import com.amazonaws.services.logs.model.PutLogEventsRequest;
import com.amazonaws.services.logs.model.PutLogEventsResult;
+import com.amazonaws.services.logs.model.ResourceNotFoundException;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.model.AuditEventBase;
@@ -61,21 +64,22 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
public static final String PROP_LOG_GROUP_NAME = "log_group";
public static final String PROP_LOG_STREAM_PREFIX = "log_stream_prefix";
public static final String CONFIG_PREFIX = "ranger.audit.amazon_cloudwatch";
+ public static final String PROP_REGION = "region";
private String logGroupName;
private String logStreamName;
private AWSLogs logsClient;
private String sequenceToken;
+ private String regionName;
@Override
public void init(Properties props, String propPrefix) {
LOG.info("init() called for CloudWatchAuditDestination");
super.init(props, propPrefix);
- this.logGroupName = MiscUtil.getStringProperty(props, propPrefix + "."
- + PROP_LOG_GROUP_NAME);
- this.logStreamName = MiscUtil.getStringProperty(props, propPrefix + "."
- + PROP_LOG_STREAM_PREFIX) + MiscUtil.generateUniqueId();
+ this.logGroupName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_GROUP_NAME, "ranger_audits");
+ this.logStreamName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_STREAM_PREFIX) + MiscUtil.generateUniqueId();
+ this.regionName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_REGION);
logsClient = getClient(); // Initialize client
createLogStream();
@@ -95,8 +99,11 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
PutLogEventsRequest req = new PutLogEventsRequest()
.withLogEvents(toInputLogEvent(collection))
.withLogGroupName(logGroupName)
- .withLogStreamName(logStreamName)
- .withSequenceToken(sequenceToken);
+ .withLogStreamName(logStreamName);
+
+ if (StringUtils.isNotBlank(sequenceToken)) {
+ req.setSequenceToken(sequenceToken);
+ }
try {
sequenceToken = pushLogEvents(req, false, client);
@@ -117,6 +124,12 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
try {
PutLogEventsResult re = client.putLogEvents(req);
sequenceToken = re.getNextSequenceToken();
+ } catch (ResourceNotFoundException ex) {
+ if (!retryingOnInvalidSeqToken) {
+ createLogStream();
+ return pushLogEvents(req, true, client);
+ }
+ throw ex;
} catch (InvalidSequenceTokenException ex) {
if (retryingOnInvalidSeqToken) {
LOG.error("Unexpected invalid sequence token. Possible race condition occurred");
@@ -124,8 +137,9 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
}
// LogStream may exist before first push attempt, re-obtain the sequence token
- LOG.info("Invalid sequence token. Plugin possibly restarted. " +
- "Updating the sequence token and retrying");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Invalid sequence token. Plugin possibly restarted. Updating the sequence token and retrying");
+ }
sequenceToken = ex.getExpectedSequenceToken();
req.setSequenceToken(sequenceToken);
return pushLogEvents(req, true, client);
@@ -159,8 +173,7 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
.withLogGroupName(logGroupName)
.withLogStreamName(logStreamName);
- LOG.info(String.format("Creating Log Stream `%s` in Log Group `%s`",
- logStreamName, logGroupName));
+ LOG.info(String.format("Creating Log Stream `%s` in Log Group `%s`", logStreamName, logGroupName));
client.createLogStream(req);
}
@@ -177,6 +190,9 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
}
private AWSLogs newClient() {
- return AWSLogsClientBuilder.standard().build();
+ if (StringUtils.isBlank(regionName)) {
+ return AWSLogsClientBuilder.standard().build();
+ }
+ return AWSLogsClientBuilder.standard().withRegion(regionName).build();
}
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
index f58b813..268cb57 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
@@ -24,6 +24,9 @@ import java.security.Principal;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
@@ -885,4 +888,73 @@ public class MiscUtil {
static final Random random = new Random();
}
+ // Utility methods
+ public static int toInt(Object value) {
+ if (value == null) {
+ return 0;
+ }
+ if (value instanceof Integer) {
+ return (Integer) value;
+ }
+ if (value.toString().isEmpty()) {
+ return 0;
+ }
+ try {
+ return Integer.valueOf(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to integer. Value = " + value, t);
+ }
+ return 0;
+ }
+
+ public static long toLong(Object value) {
+ if (value == null) {
+ return 0;
+ }
+ if (value instanceof Long) {
+ return (Long) value;
+ }
+ if (value.toString().isEmpty()) {
+ return 0;
+ }
+ try {
+ return Long.valueOf(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to long. Value = " + value, t);
+ }
+ return 0;
+ }
+
+ public static Date toDate(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date) value;
+ }
+ try {
+ // TODO: Do proper parsing based on Solr response value
+ return new Date(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to date. Value = " + value, t);
+ }
+ return null;
+ }
+
+ public static Date toLocalDate(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date) value;
+ }
+ try {
+ LocalDateTime localDateTime = LocalDateTime.parse(value.toString(), DateTimeFormatter.ISO_DATE_TIME);
+ return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant());
+ } catch (Throwable t) {
+ logger.error("Error converting value to date. Value = " + value, t);
+ }
+ return null;
+ }
+
}
diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
index a6c7ffd..a743db4 100644
--- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg
+++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
@@ -60,6 +60,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index 87a2481..d105049 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -110,6 +110,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
index 92d2a4b..8e0e158 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
+++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
@@ -57,6 +57,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 323b878..698638b 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/hive-agent/conf/ranger-hive-audit-changes.cfg
+++ b/hive-agent/conf/ranger-hive-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 3720b66..e64e5e0 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -107,6 +107,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties
index 6b6b662..4cf7908 100755
--- a/kms/scripts/install.properties
+++ b/kms/scripts/install.properties
@@ -227,6 +227,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/knox-agent/conf/ranger-knox-audit-changes.cfg
+++ b/knox-agent/conf/ranger-knox-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index 4704004..cb1ccf8 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -102,6 +102,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg
index 2d8251b..59c3927 100644
--- a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg
+++ b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg
@@ -38,6 +38,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
#log4j configuration
xasecure.audit.log4j.is.enabled %XAAUDIT.LOG4J.ENABLE% mod create-if-not-exists
diff --git a/plugin-atlas/scripts/install.properties b/plugin-atlas/scripts/install.properties
index 3b777bd..c5b6eb8 100644
--- a/plugin-atlas/scripts/install.properties
+++ b/plugin-atlas/scripts/install.properties
@@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
+++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-elasticsearch/scripts/install.properties b/plugin-elasticsearch/scripts/install.properties
index 4111afe..fb2b40b 100644
--- a/plugin-elasticsearch/scripts/install.properties
+++ b/plugin-elasticsearch/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
index bc5a089..94c2d24 100644
--- a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
+++ b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
@@ -51,6 +51,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-kafka/scripts/install.properties b/plugin-kafka/scripts/install.properties
index 1e325e0..21f718b 100644
--- a/plugin-kafka/scripts/install.properties
+++ b/plugin-kafka/scripts/install.properties
@@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-kms/conf/ranger-kms-audit-changes.cfg b/plugin-kms/conf/ranger-kms-audit-changes.cfg
index e5e9ae4..50e52a1 100644
--- a/plugin-kms/conf/ranger-kms-audit-changes.cfg
+++ b/plugin-kms/conf/ranger-kms-audit-changes.cfg
@@ -60,6 +60,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg
+++ b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-kylin/scripts/install.properties b/plugin-kylin/scripts/install.properties
index 0134338..9d117ec 100644
--- a/plugin-kylin/scripts/install.properties
+++ b/plugin-kylin/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg
index 0eace6d..a3e2382 100644
--- a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg
+++ b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg
@@ -51,6 +51,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-ozone/scripts/install.properties b/plugin-ozone/scripts/install.properties
index 1891d56..fcb74a9 100644
--- a/plugin-ozone/scripts/install.properties
+++ b/plugin-ozone/scripts/install.properties
@@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-presto/conf/ranger-presto-audit-changes.cfg b/plugin-presto/conf/ranger-presto-audit-changes.cfg
index bc5a089..94c2d24 100644
--- a/plugin-presto/conf/ranger-presto-audit-changes.cfg
+++ b/plugin-presto/conf/ranger-presto-audit-changes.cfg
@@ -51,6 +51,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-presto/scripts/install.properties b/plugin-presto/scripts/install.properties
index ce162a2..f8e8ac1 100644
--- a/plugin-presto/scripts/install.properties
+++ b/plugin-presto/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-solr/conf/ranger-solr-audit-changes.cfg b/plugin-solr/conf/ranger-solr-audit-changes.cfg
index ffa0a76..5ab43d6 100644
--- a/plugin-solr/conf/ranger-solr-audit-changes.cfg
+++ b/plugin-solr/conf/ranger-solr-audit-changes.cfg
@@ -52,6 +52,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-solr/scripts/install.properties b/plugin-solr/scripts/install.properties
index d1852e6..1c292f1 100644
--- a/plugin-solr/scripts/install.properties
+++ b/plugin-solr/scripts/install.properties
@@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg
+++ b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-sqoop/scripts/install.properties b/plugin-sqoop/scripts/install.properties
index 81b4526..33d8813 100644
--- a/plugin-sqoop/scripts/install.properties
+++ b/plugin-sqoop/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
+++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties
index e73ab8b..f7eb036 100644
--- a/plugin-yarn/scripts/install.properties
+++ b/plugin-yarn/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties
diff --git a/pom.xml b/pom.xml
index f9c46f6..5c621a5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -230,6 +230,7 @@
<joda.time.version>2.10.6</joda.time.version>
<jsonsmart.version>2.3.1</jsonsmart.version>
<nimbus-jose-jwt.version>8.22.1</nimbus-jose-jwt.version>
+ <com.amazonaws.aws-java-sdk-bom.version>1.12.125</com.amazonaws.aws-java-sdk-bom.version>
</properties>
<profiles>
<profile>
diff --git a/security-admin/pom.xml b/security-admin/pom.xml
index e9e9a53..bcce606 100644
--- a/security-admin/pom.xml
+++ b/security-admin/pom.xml
@@ -29,6 +29,17 @@
<properties>
<skipJSTests>false</skipJSTests>
</properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bom</artifactId>
+ <version>${com.amazonaws.aws-java-sdk-bom.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
<groupId>org.antlr</groupId>
@@ -674,6 +685,10 @@
<artifactId>httpcore-nio</artifactId>
<version>${httpcomponents.httpcore.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-logs</artifactId>
+ </dependency>
</dependencies>
<build>
<pluginManagement>
diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties
index 5a8b00c..03f6cd7 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -83,7 +83,7 @@ rangerUsersync_password=
keyadmin_password=
-#Source for Audit Store. Currently solr and elasticsearch are supported.
+#Source for Audit Store. Currently solr, elasticsearch and cloudwatch logs are supported.
# * audit_store is solr
audit_store=solr
@@ -113,6 +113,11 @@ audit_solr_max_shards_per_node=1
audit_solr_acl_user_list_sasl=solr,infra-solr
audit_solr_bootstrap_enabled=true
+# * audit to amazon cloudwatch properties
+audit_cloudwatch_region=
+audit_cloudwatch_log_group=
+audit_cloudwatch_log_stream_prefix=
+
#------------------------- DB CONFIG - END ----------------------------------
#
diff --git a/security-admin/scripts/ranger-admin-site-template.xml b/security-admin/scripts/ranger-admin-site-template.xml
index 72ff66e..037260f 100644
--- a/security-admin/scripts/ranger-admin-site-template.xml
+++ b/security-admin/scripts/ranger-admin-site-template.xml
@@ -177,6 +177,18 @@
<value></value>
</property>
<property>
+ <name>ranger.audit.amazon_cloudwatch.region</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.amazon_cloudwatch.log_group</name>
+ <value></value>
+ </property>
+ <property>
+ <name>ranger.audit.amazon_cloudwatch.log_stream_prefix</name>
+ <value></value>
+ </property>
+ <property>
<name>ranger.jpa.audit.jdbc.dialect</name>
<value></value>
</property>
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index c3f51a0..9c5da5e 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -85,6 +85,9 @@ audit_solr_urls=$(get_prop 'audit_solr_urls' $PROPFILE)
audit_solr_user=$(get_prop 'audit_solr_user' $PROPFILE)
audit_solr_password=$(get_prop 'audit_solr_password' $PROPFILE)
audit_solr_zookeepers=$(get_prop 'audit_solr_zookeepers' $PROPFILE)
+audit_cloudwatch_region=$(get_prop 'audit_cloudwatch_region' $PROPFILE)
+audit_cloudwatch_log_group=$(get_prop 'audit_cloudwatch_log_group' $PROPFILE)
+audit_cloudwatch_log_stream_prefix=$(get_prop 'audit_cloudwatch_log_stream_prefix' $PROPFILE)
policymgr_external_url=$(get_prop 'policymgr_external_url' $PROPFILE)
policymgr_http_enabled=$(get_prop 'policymgr_http_enabled' $PROPFILE)
policymgr_https_keystore_file=$(get_prop 'policymgr_https_keystore_file' $PROPFILE)
@@ -269,6 +272,17 @@ init_variables(){
fi
fi
+ if [ "${audit_store}" == "cloudwatch" ] ;then
+ if [ "${audit_cloudwatch_region}" == "" ] ;then
+ log "[I] Please provide valid region for 'amazon cloudwatch' audit store!"
+ exit 1
+ fi
+ if [ "${audit_cloudwatch_log_group}" == "" ] ;then
+ log "[I] Please provide valid log-group for 'amazon cloudwatch' audit store!"
+ exit 1
+ fi
+ fi
+
db_ssl_enabled=`echo $db_ssl_enabled | tr '[:upper:]' '[:lower:]'`
if [ "${db_ssl_enabled}" != "true" ]
then
@@ -800,6 +814,21 @@ update_properties() {
fi
+ if [ "${audit_store}" == "cloudwatch" ]
+ then
+ propertyName=ranger.audit.amazon_cloudwatch.region
+ newPropertyValue=${audit_cloudwatch_region}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+
+ propertyName=ranger.audit.amazon_cloudwatch.log_group
+ newPropertyValue=${audit_cloudwatch_log_group}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+
+ propertyName=ranger.audit.amazon_cloudwatch.log_stream_prefix
+ newPropertyValue=${audit_cloudwatch_log_stream_prefix}
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+ fi
+
if [ "${audit_store}" != "" ]
then
propertyName=ranger.audit.source.type
diff --git a/security-admin/scripts/upgrade_admin.py b/security-admin/scripts/upgrade_admin.py
index 10fa485..85f57b8 100755
--- a/security-admin/scripts/upgrade_admin.py
+++ b/security-admin/scripts/upgrade_admin.py
@@ -116,6 +116,9 @@ config2xmlMAP = {
'audit_elasticsearch_port':'ranger.audit.elasticsearch.port',
'audit_elasticsearch_user':'ranger.audit.elasticsearch.user',
'audit_elasticsearch_password':'ranger.audit.elasticsearch.password',
+ 'audit_cloudwatch_region':'ranger.audit.amazon_cloudwatch.region',
+ 'audit_cloudwatch_log_group':'ranger.audit.amazon_cloudwatch.log_group',
+ 'audit_cloudwatch_log_stream_prefix':'ranger.audit.amazon_cloudwatch.log_stream_prefix',
'audit_solr_urls':'ranger.audit.solr.urls',
'auditDB.jdbc.dialect':'ranger.jpa.audit.jdbc.dialect',
'auditDB.jdbc.driver':'ranger.jpa.audit.jdbc.driver',
diff --git a/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java
index 4d97f28..de1feed 100644
--- a/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java
@@ -77,6 +77,10 @@ public class AccessAuditsService {
SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
searchFields.add(new SearchField("repoType", "repoType",
SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ /* Note; search fields starting with '-' denotes exclude conditions,
+ * it should be handled manually if audit destination does not support the same.
+ * solr support this way while cloudwatch does not.
+ */
searchFields.add(new SearchField("-repoType", "-repoType",
SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
searchFields.add(new SearchField("-requestUser", "-reqUser",
diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java
new file mode 100644
index 0000000..5c18016
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.amazon.cloudwatch;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.log4j.Logger;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.common.JSONUtil;
+import org.apache.ranger.common.MessageEnums;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.entity.XXService;
+import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.view.VXAccessAudit;
+import org.apache.ranger.view.VXAccessAuditList;
+import org.apache.ranger.view.VXLong;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Service;
+
+import com.amazonaws.services.logs.AWSLogs;
+import com.amazonaws.services.logs.model.FilteredLogEvent;
+
+@Service
+@Scope("singleton")
+public class CloudWatchAccessAuditsService extends org.apache.ranger.AccessAuditsService {
+ private static final Logger LOGGER = Logger.getLogger(CloudWatchAccessAuditsService.class);
+
+ @Autowired
+ CloudWatchMgr cloudWatchMgr;
+
+ @Autowired
+ CloudWatchUtil cloudWatchUtil;
+
+ @Autowired
+ JSONUtil jsonUtil;
+
+ public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
+
+ final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true);
+ AWSLogs client = cloudWatchMgr.getClient();
+ if (client == null) {
+ LOGGER.warn("CloudWatch client is null, so not running the query.");
+ throw restErrorUtil.createRESTException("Error connecting to cloudwatch", MessageEnums.ERROR_SYSTEM);
+ }
+
+ List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
+ Map<String, Object> paramList = searchCriteria.getParamList();
+ updateUserExclusion(paramList);
+
+ List<FilteredLogEvent> result;
+ try {
+ result = cloudWatchUtil.searchResources(client, searchCriteria, searchFields, sortFields);
+ } catch (Exception e) {
+ LOGGER.warn(String.format("CloudWatch query failed: %s", e.getMessage()));
+ throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM);
+ }
+
+ VXAccessAuditList returnList = new VXAccessAuditList();
+ if (result != null && CollectionUtils.isNotEmpty(result)) {
+ int recordCount = 0;
+ int endIndex = result.size() - 1;
+ endIndex = endIndex - searchCriteria.getStartIndex() < 0 ? endIndex : endIndex - searchCriteria.getStartIndex();
+ for (int index = endIndex; recordCount < searchCriteria.getMaxRows() && index >=0 ; index--) {
+ FilteredLogEvent event = result.get(index);
+ AuthzAuditEvent auditEvent = null;
+ try {
+ auditEvent = MiscUtil.fromJson(event.getMessage(), AuthzAuditEvent.class);
+ } catch (Exception ex) {
+ LOGGER.error("Error while parsing json data" , ex);
+ }
+ VXAccessAudit vXAccessAudit = populateViewBean(auditEvent);
+ if (vXAccessAudit != null) {
+ String serviceType = vXAccessAudit.getServiceType();
+ boolean isHive = "hive".equalsIgnoreCase(serviceType);
+ if (!hiveQueryVisibility && isHive) {
+ vXAccessAudit.setRequestData(null);
+ } else if (isHive) {
+ String accessType = vXAccessAudit.getAccessType();
+ if ("grant".equalsIgnoreCase(accessType) || "revoke".equalsIgnoreCase(accessType)) {
+ String requestData = vXAccessAudit.getRequestData();
+ if (requestData != null) {
+ try {
+ vXAccessAudit.setRequestData(java.net.URLDecoder.decode(requestData, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ LOGGER.warn("Error while encoding request data: " + requestData, e);
+ }
+ } else {
+ LOGGER.warn("Error in request data of audit from cloudwatch. AuditData: "+ vXAccessAudit.toString());
+ }
+ }
+ }
+ }
+ xAccessAuditList.add(vXAccessAudit);
+ recordCount++;
+ }
+ returnList.setResultSize(result.size());
+ returnList.setTotalCount(result.size());
+ }
+
+ returnList.setPageSize(searchCriteria.getMaxRows());
+ returnList.setStartIndex(searchCriteria.getStartIndex());
+ returnList.setVXAccessAudits(xAccessAuditList);
+ return returnList;
+ }
+
+ public void setRestErrorUtil(RESTErrorUtil restErrorUtil) {
+ this.restErrorUtil = restErrorUtil;
+ }
+
+ public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
+ long count = 100;
+ VXLong vXLong = new VXLong();
+ vXLong.setValue(count);
+ return vXLong;
+ }
+
+ private VXAccessAudit populateViewBean(AuthzAuditEvent auditEvent) {
+ VXAccessAudit accessAudit = new VXAccessAudit();
+
+ Object value = null;
+ if(LOGGER.isDebugEnabled()) {
+ LOGGER.debug("doc=" + auditEvent.toString());
+ }
+
+ value = auditEvent.getEventId();
+ if (value != null) {
+ accessAudit.setId((long) value.hashCode());
+ accessAudit.setEventId(value.toString());
+ }
+
+ value = auditEvent.getClusterName();
+ if (value != null) {
+ accessAudit.setClusterName(value.toString());
+ }
+
+ value = auditEvent.getZoneName();
+ if (value != null) {
+ accessAudit.setZoneName(value.toString());
+ }
+
+ value = auditEvent.getAgentHostname();
+ if (value != null) {
+ accessAudit.setAgentHost(value.toString());
+ }
+
+ value = auditEvent.getPolicyVersion();
+ if (value != null) {
+ accessAudit.setPolicyVersion(MiscUtil.toLong(value));
+ }
+
+ value = auditEvent.getAccessType();
+ if (value != null) {
+ accessAudit.setAccessType(value.toString());
+ }
+
+ value = auditEvent.getAclEnforcer();
+ if (value != null) {
+ accessAudit.setAclEnforcer(value.toString());
+ }
+
+ value = auditEvent.getAgentId();
+ if (value != null) {
+ accessAudit.setAgentId(value.toString());
+ }
+
+ value = auditEvent.getRepositoryName();
+ if (value != null) {
+ accessAudit.setRepoName(value.toString());
+ XXService xxService = daoManager.getXXService().findByName(accessAudit.getRepoName());
+
+ if(xxService != null) {
+ accessAudit.setRepoDisplayName(xxService.getDisplayName());
+ }
+ }
+
+ value = auditEvent.getSessionId();
+ if (value != null) {
+ accessAudit.setSessionId(value.toString());
+ }
+
+ value = auditEvent.getUser();
+ if (value != null) {
+ accessAudit.setRequestUser(value.toString());
+ }
+
+ value = auditEvent.getRequestData();
+ if (value != null) {
+ accessAudit.setRequestData(value.toString());
+ }
+ value = auditEvent.getResourcePath();
+ if (value != null) {
+ accessAudit.setResourcePath(value.toString());
+ }
+
+ value = auditEvent.getClientIP();
+ if (value != null) {
+ accessAudit.setClientIP(value.toString());
+ }
+
+ value = auditEvent.getAccessResult();
+ if (value != null) {
+ accessAudit.setAccessResult(MiscUtil.toInt(value));
+ }
+
+ value = auditEvent.getPolicyId();
+ if (value != null) {
+ accessAudit.setPolicyId(MiscUtil.toLong(value));
+ }
+
+ value = auditEvent.getRepositoryType();
+ if (value != null) {
+ accessAudit.setRepoType(MiscUtil.toInt(value));
+ XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById((long) accessAudit.getRepoType());
+ if (xServiceDef != null) {
+ accessAudit.setServiceType(xServiceDef.getName());
+ accessAudit.setServiceTypeDisplayName(xServiceDef.getDisplayName());
+ }
+ }
+
+ value = auditEvent.getResourceType();
+ if (value != null) {
+ accessAudit.setResourceType(value.toString());
+ }
+
+ value = auditEvent.getResultReason();
+ if (value != null) {
+ accessAudit.setResultReason(value.toString());
+ }
+
+ value = auditEvent.getAction();
+ if (value != null) {
+ accessAudit.setAction(value.toString());
+ }
+
+ value = auditEvent.getEventTime();
+ if (value != null) {
+ accessAudit.setEventTime(MiscUtil.toLocalDate(value));
+ }
+
+ value = auditEvent.getSeqNum();
+ if (value != null) {
+ accessAudit.setSequenceNumber(MiscUtil.toLong(value));
+ }
+
+ value = auditEvent.getEventCount();
+ if (value != null) {
+ accessAudit.setEventCount(MiscUtil.toLong(value));
+ }
+
+ value = auditEvent.getEventDurationMS();
+ if (value != null) {
+ accessAudit.setEventDuration(MiscUtil.toLong(value));
+ }
+
+ value = auditEvent.getTags();
+ if (value != null) {
+ accessAudit.setTags(value.toString());
+ }
+
+ return accessAudit;
+ }
+
+}
\ No newline at end of file
diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchMgr.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchMgr.java
new file mode 100644
index 0000000..55d8229
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchMgr.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.amazon.cloudwatch;
+
+import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.CONFIG_PREFIX;
+import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.PROP_REGION;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.ranger.common.PropertiesUtil;
+import org.springframework.stereotype.Component;
+
+import com.amazonaws.services.logs.AWSLogs;
+import com.amazonaws.services.logs.AWSLogsClientBuilder;
+
+/**
+ * This class initializes the CloudWatch client
+ *
+ */
+@Component
+public class CloudWatchMgr {
+
+ private static final Logger LOGGER = Logger.getLogger(CloudWatchMgr.class);
+
+ private AWSLogs client = null;
+ private String regionName;
+
+ synchronized void connect() {
+ if (client == null) {
+ synchronized (CloudWatchMgr.class) {
+ if (client == null) {
+ regionName = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + PROP_REGION);
+ try {
+ client = newClient();
+ } catch (Throwable t) {
+ LOGGER.fatal("Can't connect to CloudWatch region: " + regionName, t);
+ }
+ }
+ }
+ }
+ }
+
+ public AWSLogs getClient() {
+ if (client == null) {
+ synchronized (CloudWatchMgr.class) {
+ if (client == null) {
+ connect();
+ }
+ }
+ }
+ return client;
+ }
+
+ private AWSLogs newClient() {
+ if (StringUtils.isBlank(regionName)) {
+ return AWSLogsClientBuilder.standard().build();
+ }
+ return AWSLogsClientBuilder.standard().withRegion(regionName).build();
+ }
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchUtil.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchUtil.java
new file mode 100644
index 0000000..599a1ab
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchUtil.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.amazon.cloudwatch;
+
+import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.CONFIG_PREFIX;
+import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.PROP_LOG_GROUP_NAME;
+import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.PROP_LOG_STREAM_PREFIX;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.SearchField;
+import org.apache.ranger.common.SearchField.SEARCH_TYPE;
+import org.apache.ranger.common.SortField;
+import org.apache.ranger.common.StringUtil;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.amazonaws.services.logs.AWSLogs;
+import com.amazonaws.services.logs.model.FilterLogEventsRequest;
+import com.amazonaws.services.logs.model.FilterLogEventsResult;
+import com.amazonaws.services.logs.model.FilteredLogEvent;
+
+@Component
+public class CloudWatchUtil {
+ private static final Logger LOGGER = Logger.getLogger(CloudWatchUtil.class);
+
+ @Autowired
+ StringUtil stringUtil;
+
+ String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+ SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr);
+ private String logGroupName;
+ private String logStreamPrefix;
+
+ public CloudWatchUtil() {
+ logGroupName = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + PROP_LOG_GROUP_NAME, "ranger_audits");
+ logStreamPrefix = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + PROP_LOG_STREAM_PREFIX, "");
+ String timeZone = PropertiesUtil.getProperty("ranger.cloudwatch.timezone");
+ if (timeZone != null) {
+ LOGGER.info("Setting timezone to " + timeZone);
+ try {
+ dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
+ } catch (Throwable t) {
+ LOGGER.error("Error setting timezone. TimeZone = " + timeZone);
+ }
+ }
+ }
+
+ public List<FilteredLogEvent> searchResources(AWSLogs client, SearchCriteria searchCriteria,
+ List<SearchField> searchFields, List<SortField> sortFieldList) {
+ List<FilteredLogEvent> result = new ArrayList<FilteredLogEvent>();
+ try {
+ String nextToken = null;
+ FilterLogEventsRequest filterLogEventsRequest = getFilterLogEventsRequest(client, searchCriteria, searchFields);
+ boolean done = false;
+ //TODO: Improve response time
+ //This approach is slow as cloudwatch doesn't provide timestamp based sorting in descending order
+ do {
+ if (nextToken != null) {
+ filterLogEventsRequest = filterLogEventsRequest.withNextToken(nextToken);
+ }
+
+ FilterLogEventsResult response = client.filterLogEvents(filterLogEventsRequest);
+ if (response != null) {
+ if (CollectionUtils.isNotEmpty(response.getEvents())) {
+ //To handle outofmemory issue, max 10k records are stored in the list
+ if (result.size() > 10000) {
+ result.clear();
+ }
+ result.addAll(response.getEvents());
+ } else {
+ done = true;
+ break;
+ }
+ // check if token is the same
+ if (response.getNextToken().equals(nextToken)) {
+ done = true;
+ break;
+ }
+ // save new token
+ nextToken = response.getNextToken();
+ if (nextToken == null) {
+ done = true;
+ break;
+ }
+ }
+ } while (!done);
+ LOGGER.info("Successfully got CloudWatch log events!");
+ } catch (Exception e) {
+ LOGGER.error("Error searching records from CloudWatch", e);
+ }
+ return result;
+ }
+
+ public FilterLogEventsRequest getFilterLogEventsRequest(AWSLogs client, SearchCriteria searchCriteria,
+ List<SearchField> searchFields) {
+ FilterLogEventsRequest filterLogEventsRequest = null;
+ StringBuilder filterPattern = new StringBuilder("");
+ Date fromDate = null;
+ Date toDate = null;
+
+ if (searchCriteria.getParamList() != null) {
+ List<String> filterExpr = new ArrayList<String>();
+
+ for (SearchField searchField : searchFields) {
+ Object paramValue = searchCriteria.getParamValue(searchField.getClientFieldName());
+ if (paramValue == null || paramValue.toString().isEmpty()) {
+ continue;
+ }
+
+ String fieldName = searchField.getFieldName();
+ if (searchField.getDataType() == SearchField.DATA_TYPE.DATE) {
+ if (!(paramValue instanceof Date)) {
+ LOGGER.error("Search field is not a Java Date Object, paramValue = " + paramValue);
+ } else {
+ if (searchField.getSearchType() == SEARCH_TYPE.GREATER_EQUAL_THAN || searchField.getSearchType() == SEARCH_TYPE.GREATER_THAN) {
+ fromDate = (Date) paramValue;
+ } else if (searchField.getSearchType() == SEARCH_TYPE.LESS_EQUAL_THAN || searchField.getSearchType() == SEARCH_TYPE.LESS_THAN) {
+ toDate = (Date) paramValue;
+ }
+ }
+ } else if (paramValue instanceof Collection) {
+ String fq = orList(fieldName, (Collection<?>) paramValue);
+ if (StringUtils.isNotBlank(fq)) {
+ filterExpr.add(fq);
+ }
+ } else {
+ String fq = null;
+ if (searchField.getSearchType() == SEARCH_TYPE.PARTIAL) {
+ fq = setFieldForPartialSearch(fieldName, paramValue);
+ } else {
+ fq = setField(fieldName, paramValue);
+ }
+ if (StringUtils.isNotBlank(fq)) {
+ filterExpr.add(fq);
+ }
+ }
+ }
+
+ if (fromDate == null) {
+ fromDate = DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH);
+ }
+ if (toDate == null) {
+ Date today = DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH);
+ toDate = DateUtils.addDays(today, 1);
+ }
+
+ // Syntax : { ($.user.id = 1) && ($.users[0].email = "user@example.com") }
+ if (CollectionUtils.isNotEmpty(filterExpr)) {
+ String strExpr = "";
+ int count = -1;
+ for (String fq : filterExpr) {
+ count++;
+ if (count > 0) {
+ strExpr += " &&";
+ }
+ strExpr = strExpr.concat("(" + fq + ")");
+ }
+ if (strExpr.endsWith("&&")) {
+ strExpr = strExpr.substring(0, strExpr.length() - 3);
+ }
+ if (StringUtils.isNotBlank(strExpr)) {
+ filterPattern.append("{" + strExpr + "}");
+ }
+ }
+ }
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("filterExpression for cloudwatch request " + filterPattern.toString());
+ }
+
+ // Add FilterPattern which will only fetch logs required
+ filterLogEventsRequest = new FilterLogEventsRequest()
+ .withLogGroupName(logGroupName)
+ .withStartTime(fromDate.getTime())
+ .withEndTime(toDate.getTime())
+ .withFilterPattern(filterPattern.toString());
+
+ if (StringUtils.isNotBlank(logStreamPrefix)) {
+ filterLogEventsRequest.setLogStreamNamePrefix(logStreamPrefix);
+ }
+
+ return filterLogEventsRequest;
+ }
+
+ //Syntax { $.user.email = "user@example.com" || $.coordinates[0][1] = nonmatch && $.actions[2] = nomatch }
+ private String orList(String fieldName, Collection<?> valueList) {
+ if (valueList == null || valueList.isEmpty()) {
+ return null;
+ }
+ String expr = "";
+ int count = -1;
+ for (Object value : valueList) {
+ count++;
+ if (count > 0) {
+ expr += " || ";
+ }
+ expr += setField(fieldName, value);
+ }
+ return expr;
+ }
+
+ private String setField(String fieldName, Object value) {
+ if (value == null || StringUtils.isBlank(value.toString())) {
+ return null;
+ }
+ if (value instanceof Integer || value instanceof Long) {
+ if (fieldName.startsWith("-")) {
+ fieldName = fieldName.substring(1);
+ return "$." + fieldName + " != " + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase());
+ }
+ return "$." + fieldName + " = " + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase());
+ }
+ if (fieldName.startsWith("-")) {
+ fieldName = fieldName.substring(1);
+ return "$." + fieldName + " != \"" + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()) + "\"";
+ }
+ return "$." + fieldName + " = \"" + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()) + "\"";
+ }
+
+ private String setFieldForPartialSearch(String fieldName, Object value) {
+ if (value == null || StringUtils.isBlank(value.toString())) {
+ return null;
+ }
+ return "$." + fieldName + "= \"*" + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()) + "*\"";
+ }
+
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
index d3ce251..36f137e 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
@@ -38,6 +38,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
+import org.apache.ranger.amazon.cloudwatch.CloudWatchAccessAuditsService;
import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.DateUtil;
import org.apache.ranger.common.JSONUtil;
@@ -114,6 +115,9 @@ public class AssetMgr extends AssetMgrBase {
ElasticSearchAccessAuditsService elasticSearchAccessAuditsService;
@Autowired
+ CloudWatchAccessAuditsService cloudWatchAccessAuditsService;
+
+ @Autowired
XPolicyService xPolicyService;
@Autowired
@@ -1128,6 +1132,8 @@ public class AssetMgr extends AssetMgrBase {
return solrAccessAuditsService.searchXAccessAudits(searchCriteria);
} else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(xaBizUtil.getAuditDBType())) {
return elasticSearchAccessAuditsService.searchXAccessAudits(searchCriteria);
+ } else if (RangerBizUtil.AUDIT_STORE_CloudWatch.equalsIgnoreCase(xaBizUtil.getAuditDBType())) {
+ return cloudWatchAccessAuditsService.searchXAccessAudits(searchCriteria);
} else {
return xAccessAuditService.searchXAccessAudits(searchCriteria);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
index 75ebae6..2d6aa41 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
@@ -111,6 +111,7 @@ public class RangerBizUtil {
public static final String AUDIT_STORE_RDBMS = "DB";
public static final String AUDIT_STORE_SOLR = "solr";
public static final String AUDIT_STORE_ElasticSearch = "elasticSearch";
+ public static final String AUDIT_STORE_CloudWatch = "cloudwatch";
public static final boolean batchClearEnabled = PropertiesUtil.getBooleanProperty("ranger.jpa.jdbc.batch-clear.enable", true);
public static final int policyBatchSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10);
public static final int batchPersistSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-persist.size", 500);
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
index 4e5410e..9354350 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
@@ -21,6 +21,7 @@ package org.apache.ranger.biz;
import javax.servlet.http.HttpServletResponse;
+import org.apache.ranger.amazon.cloudwatch.CloudWatchAccessAuditsService;
import org.apache.ranger.common.ContextUtil;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.UserSessionBase;
@@ -45,6 +46,9 @@ public class XAuditMgr extends XAuditMgrBase {
ElasticSearchAccessAuditsService elasticSearchAccessAuditsService;
@Autowired
+ CloudWatchAccessAuditsService cloudWatchAccessAuditsService;
+
+ @Autowired
RangerBizUtil rangerBizUtil;
public VXTrxLog getXTrxLog(Long id) {
@@ -119,6 +123,8 @@ public class XAuditMgr extends XAuditMgrBase {
return solrAccessAuditsService.searchXAccessAudits(searchCriteria);
} else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(auditDBType)) {
return elasticSearchAccessAuditsService.searchXAccessAudits(searchCriteria);
+ } else if (RangerBizUtil.AUDIT_STORE_CloudWatch.equalsIgnoreCase(auditDBType)) {
+ return cloudWatchAccessAuditsService.searchXAccessAudits(searchCriteria);
} else {
return super.searchXAccessAudits(searchCriteria);
}
@@ -131,6 +137,8 @@ public class XAuditMgr extends XAuditMgrBase {
return solrAccessAuditsService.getXAccessAuditSearchCount(searchCriteria);
} else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(auditDBType)) {
return elasticSearchAccessAuditsService.getXAccessAuditSearchCount(searchCriteria);
+ } else if (RangerBizUtil.AUDIT_STORE_CloudWatch.equalsIgnoreCase(auditDBType)) {
+ return cloudWatchAccessAuditsService.getXAccessAuditSearchCount(searchCriteria);
} else {
return super.getXAccessAuditSearchCount(searchCriteria);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
index 0b2e7df..5db858d 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
@@ -20,6 +20,7 @@
package org.apache.ranger.elasticsearch;
import org.apache.log4j.Logger;
+import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.common.RESTErrorUtil;
@@ -169,7 +170,7 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu
value = source.get("policyVersion");
if (value != null) {
- accessAudit.setPolicyVersion(elasticSearchUtil.toLong(value));
+ accessAudit.setPolicyVersion(MiscUtil.toLong(value));
}
value = source.get("access");
@@ -221,15 +222,15 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu
//}
value = source.get("result");
if (value != null) {
- accessAudit.setAccessResult(elasticSearchUtil.toInt(value));
+ accessAudit.setAccessResult(MiscUtil.toInt(value));
}
value = source.get("policy");
if (value != null) {
- accessAudit.setPolicyId(elasticSearchUtil.toLong(value));
+ accessAudit.setPolicyId(MiscUtil.toLong(value));
}
value = source.get("repoType");
if (value != null) {
- accessAudit.setRepoType(elasticSearchUtil.toInt(value));
+ accessAudit.setRepoType(MiscUtil.toInt(value));
if(null != daoManager) {
XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef();
if(xxServiceDef != null) {
@@ -255,19 +256,19 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu
}
value = source.get("evtTime");
if (value != null) {
- accessAudit.setEventTime(elasticSearchUtil.toDate(value));
+ accessAudit.setEventTime(MiscUtil.toDate(value));
}
value = source.get("seq_num");
if (value != null) {
- accessAudit.setSequenceNumber(elasticSearchUtil.toLong(value));
+ accessAudit.setSequenceNumber(MiscUtil.toLong(value));
}
value = source.get("event_count");
if (value != null) {
- accessAudit.setEventCount(elasticSearchUtil.toLong(value));
+ accessAudit.setEventCount(MiscUtil.toLong(value));
}
value = source.get("event_dur_ms");
if (value != null) {
- accessAudit.setEventDuration(elasticSearchUtil.toLong(value));
+ accessAudit.setEventDuration(MiscUtil.toLong(value));
}
value = source.get("tags");
if (value != null) {
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
index 9bee640..777029e 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
@@ -68,60 +68,6 @@ public class ElasticSearchUtil {
}
}
-
- // Utility methods
- public int toInt(Object value) {
- if (value == null) {
- return 0;
- }
- if (value instanceof Integer) {
- return (Integer) value;
- }
- if (value.toString().isEmpty()) {
- return 0;
- }
- try {
- return Integer.valueOf(value.toString());
- } catch (Throwable t) {
- logger.error("Error converting value to integer. Value = " + value, t);
- }
- return 0;
- }
-
- public long toLong(Object value) {
- if (value == null) {
- return 0;
- }
- if (value instanceof Long) {
- return (Long) value;
- }
- if (value.toString().isEmpty()) {
- return 0;
- }
- try {
- return Long.valueOf(value.toString());
- } catch (Throwable t) {
- logger.error("Error converting value to long. Value = " + value, t);
- }
- return 0;
- }
-
- public Date toDate(Object value) {
- if (value == null) {
- return null;
- }
- if (value instanceof Date) {
- return (Date) value;
- }
- try {
- LocalDateTime localDateTime = LocalDateTime.parse(value.toString(), DateTimeFormatter.ISO_DATE_TIME);
- return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant());
- } catch (Throwable t) {
- logger.error("Error converting value to date. Value = " + value, t);
- }
- return null;
- }
-
public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException {
// See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html
QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria);
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
index 0aea46d..2f6c869 100644
--- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.ranger.AccessAuditsService;
+import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.common.RESTErrorUtil;
@@ -153,7 +154,7 @@ public class SolrAccessAuditsService extends AccessAuditsService {
value = doc.getFieldValue("policyVersion");
if (value != null) {
- accessAudit.setPolicyVersion(solrUtil.toLong(value));
+ accessAudit.setPolicyVersion(MiscUtil.toLong(value));
}
value = doc.getFieldValue("access");
@@ -205,15 +206,15 @@ public class SolrAccessAuditsService extends AccessAuditsService {
//}
value = doc.getFieldValue("result");
if (value != null) {
- accessAudit.setAccessResult(solrUtil.toInt(value));
+ accessAudit.setAccessResult(MiscUtil.toInt(value));
}
value = doc.getFieldValue("policy");
if (value != null) {
- accessAudit.setPolicyId(solrUtil.toLong(value));
+ accessAudit.setPolicyId(MiscUtil.toLong(value));
}
value = doc.getFieldValue("repoType");
if (value != null) {
- accessAudit.setRepoType(solrUtil.toInt(value));
+ accessAudit.setRepoType(MiscUtil.toInt(value));
XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById((long) accessAudit.getRepoType());
if (xServiceDef != null) {
accessAudit.setServiceType(xServiceDef.getName());
@@ -234,19 +235,19 @@ public class SolrAccessAuditsService extends AccessAuditsService {
}
value = doc.getFieldValue("evtTime");
if (value != null) {
- accessAudit.setEventTime(solrUtil.toDate(value));
+ accessAudit.setEventTime(MiscUtil.toDate(value));
}
value = doc.getFieldValue("seq_num");
if (value != null) {
- accessAudit.setSequenceNumber(solrUtil.toLong(value));
+ accessAudit.setSequenceNumber(MiscUtil.toLong(value));
}
value = doc.getFieldValue("event_count");
if (value != null) {
- accessAudit.setEventCount(solrUtil.toLong(value));
+ accessAudit.setEventCount(MiscUtil.toLong(value));
}
value = doc.getFieldValue("event_dur_ms");
if (value != null) {
- accessAudit.setEventDuration(solrUtil.toLong(value));
+ accessAudit.setEventDuration(MiscUtil.toLong(value));
}
value = doc.getFieldValue("tags");
if (value != null) {
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java
index 239698f..aa80e20 100644
--- a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java
@@ -302,56 +302,4 @@ public class SolrUtil {
}
}
- // Utility methods
- public int toInt(Object value) {
- if (value == null) {
- return 0;
- }
- if (value instanceof Integer) {
- return (Integer) value;
- }
- if (value.toString().isEmpty()) {
- return 0;
- }
- try {
- return Integer.valueOf(value.toString());
- } catch (Throwable t) {
- logger.error("Error converting value to integer. Value = " + value, t);
- }
- return 0;
- }
-
- public long toLong(Object value) {
- if (value == null) {
- return 0;
- }
- if (value instanceof Long) {
- return (Long) value;
- }
- if (value.toString().isEmpty()) {
- return 0;
- }
- try {
- return Long.valueOf(value.toString());
- } catch (Throwable t) {
- logger.error("Error converting value to long. Value = " + value, t);
- }
- return 0;
- }
-
- public Date toDate(Object value) {
- if (value == null) {
- return null;
- }
- if (value instanceof Date) {
- return (Date) value;
- }
- try {
- // TODO: Do proper parsing based on Solr response value
- return new Date(value.toString());
- } catch (Throwable t) {
- logger.error("Error converting value to date. Value = " + value, t);
- }
- return null;
- }
}
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
index d32a324..839cf18 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
@@ -75,6 +75,18 @@
<value>true</value>
</property>
<property>
+ <name>ranger.audit.amazon_cloudwatch.region</name>
+ <value>us-east-2</value>
+ </property>
+ <property>
+ <name>ranger.audit.amazon_cloudwatch.log_group</name>
+ <value>ranger_audits</value>
+ </property>
+ <property>
+ <name>ranger.audit.amazon_cloudwatch.log_stream_prefix</name>
+ <value></value>
+ </property>
+ <property>
<name>ranger.audit.solr.urls</name>
<value>http://##solr_host##:6083/solr/ranger_audits</value>
<description></description>
diff --git a/storm-agent/conf/ranger-storm-audit-changes.cfg b/storm-agent/conf/ranger-storm-audit-changes.cfg
index 52c715e..ec98baf 100644
--- a/storm-agent/conf/ranger-storm-audit-changes.cfg
+++ b/storm-agent/conf/ranger-storm-audit-changes.cfg
@@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch
xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists
xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists
xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists
xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index d219abf..58be387 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
# End of V3 properties