You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2024/12/06 07:08:37 UTC
(ranger) branch RANGER-5018 created (now c16d88e5b)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a change to branch RANGER-5018
in repository https://gitbox.apache.org/repos/asf/ranger.git
at c16d88e5b RANGER-5018: checkstyle complince updates - agents-audit
This branch includes the following new commits:
new c16d88e5b RANGER-5018: checkstyle complince updates - agents-audit
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(ranger) 01/01: RANGER-5018: checkstyle complince updates - agents-audit
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch RANGER-5018
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit c16d88e5b16a628eba126f3bfe0eb472118ccbfa
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Thu Dec 5 14:09:26 2024 -0800
RANGER-5018: checkstyle complince updates - agents-audit
---
agents-audit/pom.xml | 1 +
.../AmazonCloudWatchAuditDestination.java | 109 +-
.../ranger/audit/destination/AuditDestination.java | 80 +-
.../destination/ElasticSearchAuditDestination.java | 307 ++--
.../audit/destination/FileAuditDestination.java | 437 ++---
.../audit/destination/HDFSAuditDestination.java | 361 ++--
.../audit/destination/Log4JAuditDestination.java | 202 ++-
.../audit/destination/SolrAuditDestination.java | 891 ++++-----
.../apache/ranger/audit/model/AuditEventBase.java | 14 +-
.../ranger/audit/model/AuditIndexRecord.java | 226 ++-
.../apache/ranger/audit/model/AuthzAuditEvent.java | 1100 ++++++-----
.../ranger/audit/model/EnumRepositoryType.java | 31 +-
.../ranger/audit/model/SPOOL_FILE_STATUS.java | 2 +-
.../ranger/audit/provider/AsyncAuditProvider.java | 526 +++---
.../audit/provider/AuditFileCacheProvider.java | 62 +-
.../apache/ranger/audit/provider/AuditHandler.java | 26 +-
.../audit/provider/AuditMessageException.java | 66 +-
.../audit/provider/AuditProviderFactory.java | 1038 +++++------
.../ranger/audit/provider/AuditWriterFactory.java | 70 +-
.../ranger/audit/provider/BaseAuditHandler.java | 935 +++++-----
.../audit/provider/BufferedAuditProvider.java | 198 +-
.../apache/ranger/audit/provider/DebugTracer.java | 23 +-
.../ranger/audit/provider/DummyAuditProvider.java | 184 +-
.../ranger/audit/provider/LocalFileLogBuffer.java | 1277 +++++++------
.../ranger/audit/provider/Log4jAuditProvider.java | 144 +-
.../apache/ranger/audit/provider/Log4jTracer.java | 56 +-
.../apache/ranger/audit/provider/LogBuffer.java | 11 +-
.../ranger/audit/provider/LogDestination.java | 28 +-
.../org/apache/ranger/audit/provider/MiscUtil.java | 1908 ++++++++++----------
.../audit/provider/MultiDestAuditProvider.java | 417 +++--
.../provider/StandAloneAuditProviderFactory.java | 53 +-
.../audit/provider/hdfs/HdfsAuditProvider.java | 124 +-
.../audit/provider/hdfs/HdfsLogDestination.java | 923 +++++-----
.../audit/provider/kafka/KafkaAuditProvider.java | 319 ++--
.../audit/provider/solr/SolrAuditProvider.java | 535 +++---
.../apache/ranger/audit/queue/AuditAsyncQueue.java | 357 ++--
.../apache/ranger/audit/queue/AuditBatchQueue.java | 679 +++----
.../audit/queue/AuditFileCacheProviderSpool.java | 1009 +++++------
.../apache/ranger/audit/queue/AuditFileQueue.java | 46 +-
.../ranger/audit/queue/AuditFileQueueSpool.java | 986 +++++-----
.../apache/ranger/audit/queue/AuditFileSpool.java | 1579 ++++++++--------
.../org/apache/ranger/audit/queue/AuditQueue.java | 385 ++--
.../ranger/audit/queue/AuditSummaryQueue.java | 469 ++---
.../org/apache/ranger/audit/test/TestEvents.java | 49 +-
.../ranger/audit/utils/AbstractKerberosUser.java | 75 +-
.../audit/utils/AbstractRangerAuditWriter.java | 285 +--
.../audit/utils/InMemoryJAASConfiguration.java | 140 +-
.../apache/ranger/audit/utils/KerberosAction.java | 20 +-
.../ranger/audit/utils/KerberosJAASConfigUser.java | 19 +-
.../apache/ranger/audit/utils/KerberosUser.java | 7 +-
.../org/apache/ranger/audit/utils/ORCFileUtil.java | 335 ++--
.../ranger/audit/utils/RangerAuditWriter.java | 2 +-
.../ranger/audit/utils/RangerJSONAuditWriter.java | 157 +-
.../ranger/audit/utils/RangerORCAuditWriter.java | 149 +-
.../apache/ranger/audit/utils/RollingTimeUtil.java | 494 ++---
.../org/apache/ranger/audit/utils/SolrAppUtil.java | 11 +-
.../audit/utils/RangerJSONAuditWriterTest.java | 52 +-
dev-support/checkstyle-suppressions.xml | 4 +
58 files changed, 10223 insertions(+), 9770 deletions(-)
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index e9c2d4b38..ec0f124f0 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -28,6 +28,7 @@
<name>Audit Component</name>
<description>Auth Audit</description>
<properties>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<securesm.version>1.2</securesm.version>
</properties>
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java
index f3ba77a3e..1707da8cc 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java
@@ -19,11 +19,6 @@
package org.apache.ranger.audit.destination;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.Properties;
-import java.util.stream.Collectors;
-
import com.amazonaws.services.logs.AWSLogs;
import com.amazonaws.services.logs.AWSLogsClientBuilder;
import com.amazonaws.services.logs.model.CreateLogStreamRequest;
@@ -32,7 +27,6 @@ import com.amazonaws.services.logs.model.InvalidSequenceTokenException;
import com.amazonaws.services.logs.model.PutLogEventsRequest;
import com.amazonaws.services.logs.model.PutLogEventsResult;
import com.amazonaws.services.logs.model.ResourceNotFoundException;
-
import org.apache.commons.lang.StringUtils;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.provider.MiscUtil;
@@ -41,6 +35,11 @@ import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.ThreadSafe;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
/**
* Writes audit events to Amazon CloudWatch Logs.
* <p>
@@ -48,52 +47,72 @@ import javax.annotation.concurrent.ThreadSafe;
* <p>
* Thread-safety is ensured by making the log method synchronized.
* This is to avoid possible race condition on {@link #sequenceToken} which is required in PutLogEvents API.
+ *
* @see <a href="https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html">PutLogEvents API Reference</a>
* <p>
* Note: Amazon CloudWatch has limits on the payload size and request rate.
* Based on the traffic, adjust the batch size and flush interval accordingly.
* <p>
- *
* @see <a href="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/cloudwatch_limits_cwl.html">Amazon CloudWatch Logs Service Limits</a>
*/
@ThreadSafe
public class AmazonCloudWatchAuditDestination extends AuditDestination {
-
private static final Logger LOG = LoggerFactory.getLogger(AmazonCloudWatchAuditDestination.class);
- public static final String PROP_LOG_GROUP_NAME = "log_group";
+ public static final String PROP_LOG_GROUP_NAME = "log_group";
public static final String PROP_LOG_STREAM_PREFIX = "log_stream_prefix";
- public static final String CONFIG_PREFIX = "ranger.audit.amazon_cloudwatch";
- public static final String PROP_REGION = "region";
+ public static final String CONFIG_PREFIX = "ranger.audit.amazon_cloudwatch";
+ public static final String PROP_REGION = "region";
- private String logGroupName;
- private String logStreamName;
- private AWSLogs logsClient;
- private String sequenceToken;
- private String regionName;
+ private String logGroupName;
+ private String logStreamName;
+ private volatile AWSLogs logsClient;
+ private String sequenceToken;
+ private String regionName;
+
+ static Collection<InputLogEvent> toInputLogEvent(Collection<AuditEventBase> collection) {
+ return collection.stream()
+ .map(e -> new InputLogEvent()
+ .withMessage(MiscUtil.stringify(e))
+ .withTimestamp(e.getEventTime().getTime()))
+ .sorted(Comparator.comparingLong(InputLogEvent::getTimestamp))
+ .collect(Collectors.toList());
+ }
@Override
public void init(Properties props, String propPrefix) {
LOG.info("init() called for CloudWatchAuditDestination");
+
super.init(props, propPrefix);
- this.logGroupName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_GROUP_NAME, "ranger_audits");
+ this.logGroupName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_GROUP_NAME, "ranger_audits");
this.logStreamName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_STREAM_PREFIX) + MiscUtil.generateUniqueId();
- this.regionName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_REGION);
+ this.regionName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_REGION);
logsClient = getClient(); // Initialize client
+
createLogStream();
}
@Override
public void stop() {
super.stop();
+
logStatus();
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ }
+
@Override
public synchronized boolean log(Collection<AuditEventBase> collection) {
- boolean ret = false;
+ boolean ret = false;
AWSLogs client = getClient();
PutLogEventsRequest req = new PutLogEventsRequest()
@@ -107,32 +126,38 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
try {
sequenceToken = pushLogEvents(req, false, client);
+
addSuccessCount(collection.size());
+
ret = true;
} catch (Throwable e) {
addFailedCount(collection.size());
+
LOG.error("Failed to send audit events", e);
}
return ret;
}
- private String pushLogEvents(PutLogEventsRequest req,
- boolean retryingOnInvalidSeqToken,
- AWSLogs client) {
+ private String pushLogEvents(PutLogEventsRequest req, boolean retryingOnInvalidSeqToken, AWSLogs client) {
String sequenceToken;
+
try {
PutLogEventsResult re = client.putLogEvents(req);
+
sequenceToken = re.getNextSequenceToken();
} catch (ResourceNotFoundException ex) {
if (!retryingOnInvalidSeqToken) {
createLogStream();
+
return pushLogEvents(req, true, client);
}
+
throw ex;
} catch (InvalidSequenceTokenException ex) {
if (retryingOnInvalidSeqToken) {
LOG.error("Unexpected invalid sequence token. Possible race condition occurred");
+
throw ex;
}
@@ -140,59 +165,51 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination {
if (LOG.isDebugEnabled()) {
LOG.debug("Invalid sequence token. Plugin possibly restarted. Updating the sequence token and retrying");
}
+
sequenceToken = ex.getExpectedSequenceToken();
+
req.setSequenceToken(sequenceToken);
+
return pushLogEvents(req, true, client);
}
return sequenceToken;
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#flush()
- */
- @Override
- public void flush() {
-
- }
-
- static Collection<InputLogEvent> toInputLogEvent(Collection<AuditEventBase> collection) {
- return collection.stream()
- .map(e -> new InputLogEvent()
- .withMessage(MiscUtil.stringify(e))
- .withTimestamp(e.getEventTime().getTime()))
- .sorted(Comparator.comparingLong(InputLogEvent::getTimestamp))
- .collect(Collectors.toList());
- }
-
private void createLogStream() {
AWSLogs client = getClient();
+
CreateLogStreamRequest req = new CreateLogStreamRequest()
.withLogGroupName(logGroupName)
.withLogStreamName(logStreamName);
- LOG.info(String.format("Creating Log Stream `%s` in Log Group `%s`", logStreamName, logGroupName));
+ LOG.info("Creating Log Stream `{}` in Log Group `{}`", logStreamName, logGroupName);
+
client.createLogStream(req);
}
private AWSLogs getClient() {
- if (logsClient == null) {
+ AWSLogs ret = logsClient;
+
+ if (ret == null) {
synchronized (AmazonCloudWatchAuditDestination.class) {
- if (logsClient == null) {
- logsClient = newClient();
+ ret = logsClient;
+
+ if (ret == null) {
+ ret = newClient();
+ logsClient = ret;
}
}
}
- return logsClient;
+ return ret;
}
private AWSLogs newClient() {
if (StringUtils.isBlank(regionName)) {
return AWSLogsClientBuilder.standard().build();
}
+
return AWSLogsClientBuilder.standard().withRegion(regionName).build();
}
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AuditDestination.java
index c221487c2..3c7654cbd 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AuditDestination.java
@@ -19,63 +19,57 @@
package org.apache.ranger.audit.destination;
-import java.util.Properties;
-
import org.apache.ranger.audit.provider.BaseAuditHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Properties;
+
/**
* This class needs to be extended by anyone who wants to build custom
* destination
*/
public abstract class AuditDestination extends BaseAuditHandler {
- private static final Logger logger = LoggerFactory.getLogger(AuditDestination.class);
-
- public AuditDestination() {
- logger.info("AuditDestination() enter");
- }
+ private static final Logger logger = LoggerFactory.getLogger(AuditDestination.class);
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#init(java.util.Properties,
- * java.lang.String)
- */
- @Override
- public void init(Properties prop, String basePropertyName) {
- super.init(prop, basePropertyName);
- }
+ public AuditDestination() {
+ logger.info("AuditDestination() enter");
+ }
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#flush()
- */
- @Override
- public void flush() {
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#init(java.util.Properties,
+ * java.lang.String)
+ */
+ @Override
+ public void init(Properties prop, String basePropertyName) {
+ super.init(prop, basePropertyName);
+ }
- }
+ @Override
+ public void start() {
+ }
- @Override
- public void start() {
-
- }
+ @Override
+ public void stop() {
+ }
- @Override
- public void stop() {
-
- }
+ @Override
+ public void waitToComplete() {
+ }
- @Override
- public void waitToComplete() {
-
- }
+ @Override
+ public void waitToComplete(long timeout) {
+ }
- @Override
- public void waitToComplete(long timeout) {
-
- }
-
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
index 8324d998b..c44e31211 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
@@ -19,21 +19,6 @@
package org.apache.ranger.audit.destination;
-import java.io.File;
-import java.security.PrivilegedActionException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.http.HttpHost;
@@ -63,91 +48,166 @@ import org.slf4j.LoggerFactory;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosTicket;
+import java.io.File;
+import java.security.PrivilegedActionException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
public class ElasticSearchAuditDestination extends AuditDestination {
private static final Logger LOG = LoggerFactory.getLogger(ElasticSearchAuditDestination.class);
- public static final String CONFIG_URLS = "urls";
- public static final String CONFIG_PORT = "port";
- public static final String CONFIG_USER = "user";
- public static final String CONFIG_PWRD = "password";
+ public static final String CONFIG_URLS = "urls";
+ public static final String CONFIG_PORT = "port";
+ public static final String CONFIG_USER = "user";
+ public static final String CONFIG_PWRD = "password";
public static final String CONFIG_PROTOCOL = "protocol";
- public static final String CONFIG_INDEX = "index";
- public static final String CONFIG_PREFIX = "ranger.audit.elasticsearch";
- public static final String DEFAULT_INDEX = "ranger_audits";
-
- private String index = CONFIG_INDEX;
- private final AtomicReference<RestHighLevelClient> clientRef = new AtomicReference<>(null);
- private String protocol;
- private String user;
- private int port;
- private String password;
- private String hosts;
+ public static final String CONFIG_INDEX = "index";
+ public static final String CONFIG_PREFIX = "ranger.audit.elasticsearch";
+ public static final String DEFAULT_INDEX = "ranger_audits";
+
+ private final AtomicReference<RestHighLevelClient> clientRef = new AtomicReference<>(null);
+ private final AtomicLong lastLoggedAt = new AtomicLong(0);
+
+ private String index = CONFIG_INDEX;
+ private String protocol;
+ private String user;
+ private int port;
+ private String password;
+ private String hosts;
private Subject subject;
public ElasticSearchAuditDestination() {
propPrefix = CONFIG_PREFIX;
}
+ public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) {
+ RestClientBuilder restClientBuilder = RestClient.builder(MiscUtil.toArray(urls, ",").stream().map(x -> new HttpHost(x, port, protocol)).toArray(HttpHost[]::new));
+ ThreadFactory clientThreadFactory = new ThreadFactoryBuilder().setNameFormat("ElasticSearch rest client %s").setDaemon(true).build();
+
+ if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) {
+ if (password.contains("keytab") && new File(password).exists()) {
+ final KerberosCredentialsProvider credentialsProvider = CredentialsProviderUtil.getKerberosCredentials(user, password);
+ final Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create().register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
+
+ restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
+ clientBuilder.setThreadFactory(clientThreadFactory);
+ clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+ clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+
+ return clientBuilder;
+ });
+ } else {
+ final CredentialsProvider credentialsProvider = CredentialsProviderUtil.getBasicCredentials(user, password);
+
+ restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
+ clientBuilder.setThreadFactory(clientThreadFactory);
+ clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+
+ return clientBuilder;
+ });
+ }
+ } else {
+ LOG.error("ElasticSearch Credentials not provided!!");
+ final CredentialsProvider credentialsProvider = null;
+
+ restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
+ clientBuilder.setThreadFactory(clientThreadFactory);
+ clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+
+ return clientBuilder;
+ });
+ }
+
+ return restClientBuilder;
+ }
@Override
public void init(Properties props, String propPrefix) {
super.init(props, propPrefix);
+
this.protocol = getStringProperty(props, propPrefix + "." + CONFIG_PROTOCOL, "http");
- this.user = getStringProperty(props, propPrefix + "." + CONFIG_USER, "");
+ this.user = getStringProperty(props, propPrefix + "." + CONFIG_USER, "");
this.password = getStringProperty(props, propPrefix + "." + CONFIG_PWRD, "");
- this.port = MiscUtil.getIntProperty(props, propPrefix + "." + CONFIG_PORT, 9200);
- this.index = getStringProperty(props, propPrefix + "." + CONFIG_INDEX, DEFAULT_INDEX);
- this.hosts = getHosts();
- LOG.info("Connecting to ElasticSearch: " + connectionString());
- getClient(); // Initialize client
- }
+ this.port = MiscUtil.getIntProperty(props, propPrefix + "." + CONFIG_PORT, 9200);
+ this.index = getStringProperty(props, propPrefix + "." + CONFIG_INDEX, DEFAULT_INDEX);
+ this.hosts = getHosts();
- private String connectionString() {
- return String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, hosts, port, index);
+ LOG.info("Connecting to ElasticSearch: {}", connectionString());
+
+ getClient(); // Initialize client
}
@Override
public void stop() {
super.stop();
+
logStatus();
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ // Empty flush method
+ }
+
@Override
public boolean log(Collection<AuditEventBase> events) {
boolean ret = false;
+
try {
logStatusIfRequired();
addTotalCount(events.size());
RestHighLevelClient client = getClient();
+
if (null == client) {
// ElasticSearch is still not initialized. So need return error
addDeferredCount(events.size());
+
return ret;
}
- ArrayList<AuditEventBase> eventList = new ArrayList<>(events);
- BulkRequest bulkRequest = new BulkRequest();
+ ArrayList<AuditEventBase> eventList = new ArrayList<>(events);
+ BulkRequest bulkRequest = new BulkRequest();
+
try {
eventList.forEach(event -> {
- AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
- String id = authzEvent.getEventId();
- Map<String, Object> doc = toDoc(authzEvent);
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+ String id = authzEvent.getEventId();
+ Map<String, Object> doc = toDoc(authzEvent);
+
bulkRequest.add(new IndexRequest(index).id(id).source(doc));
});
} catch (Exception ex) {
addFailedCount(eventList.size());
logFailedEvent(eventList, ex);
}
+
BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT);
if (response.status().getStatus() >= 400) {
addFailedCount(eventList.size());
logFailedEvent(eventList, "HTTP " + response.status().getStatus());
} else {
BulkItemResponse[] items = response.getItems();
+
for (int i = 0; i < items.length; i++) {
- AuditEventBase itemRequest = eventList.get(i);
+ AuditEventBase itemRequest = eventList.get(i);
BulkItemResponse itemResponse = items[i];
+
if (itemResponse.isFailed()) {
addFailedCount(1);
logFailedEvent(Arrays.asList(itemRequest), itemResponse.getFailureMessage());
@@ -155,105 +215,102 @@ public class ElasticSearchAuditDestination extends AuditDestination {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Indexed %s", itemRequest.getEventKey()));
}
+
addSuccessCount(1);
+
ret = true;
}
}
}
} catch (Throwable t) {
addDeferredCount(events.size());
+
logError("Error sending message to ElasticSearch", t);
}
return ret;
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#flush()
- */
- @Override
- public void flush() {
- // Empty flush method
- }
-
public boolean isAsync() {
return true;
}
synchronized RestHighLevelClient getClient() {
RestHighLevelClient client = clientRef.get();
+
if (client == null) {
synchronized (ElasticSearchAuditDestination.class) {
client = clientRef.get();
+
if (client == null) {
client = newClient();
+
clientRef.set(client);
}
}
}
+
if (subject != null) {
KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject);
+
try {
if (new Date().getTime() > ticket.getEndTime().getTime()) {
clientRef.set(null);
+
CredentialsProviderUtil.ticketExpireTime80 = 0;
+
client = newClient();
+
clientRef.set(client);
} else if (CredentialsProviderUtil.ticketWillExpire(ticket)) {
subject = CredentialsProviderUtil.login(user, password);
}
} catch (PrivilegedActionException e) {
LOG.error("PrivilegedActionException:", e);
+
throw new RuntimeException(e);
}
}
+
return client;
}
- private final AtomicLong lastLoggedAt = new AtomicLong(0);
+ Map<String, Object> toDoc(AuthzAuditEvent auditEvent) {
+ Map<String, Object> doc = new HashMap<>();
- public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) {
- RestClientBuilder restClientBuilder = RestClient.builder(
- MiscUtil.toArray(urls, ",").stream()
- .map(x -> new HttpHost(x, port, protocol))
- .toArray(HttpHost[]::new)
- );
- ThreadFactory clientThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat("ElasticSearch rest client %s")
- .setDaemon(true)
- .build();
- if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) {
- if (password.contains("keytab") && new File(password).exists()) {
- final KerberosCredentialsProvider credentialsProvider =
- CredentialsProviderUtil.getKerberosCredentials(user, password);
- Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
- .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
- restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
- clientBuilder.setThreadFactory(clientThreadFactory);
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
- return clientBuilder;
- });
- } else {
- final CredentialsProvider credentialsProvider =
- CredentialsProviderUtil.getBasicCredentials(user, password);
- restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
- clientBuilder.setThreadFactory(clientThreadFactory);
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- return clientBuilder;
- });
- }
- } else {
- LOG.error("ElasticSearch Credentials not provided!!");
- final CredentialsProvider credentialsProvider = null;
- restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
- clientBuilder.setThreadFactory(clientThreadFactory);
- clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
- return clientBuilder;
- });
- }
- return restClientBuilder;
+ doc.put("id", auditEvent.getEventId());
+ doc.put("access", auditEvent.getAccessType());
+ doc.put("enforcer", auditEvent.getAclEnforcer());
+ doc.put("agent", auditEvent.getAgentId());
+ doc.put("repo", auditEvent.getRepositoryName());
+ doc.put("sess", auditEvent.getSessionId());
+ doc.put("reqUser", auditEvent.getUser());
+ doc.put("reqData", auditEvent.getRequestData());
+ doc.put("resource", auditEvent.getResourcePath());
+ doc.put("cliIP", auditEvent.getClientIP());
+ doc.put("logType", auditEvent.getLogType());
+ doc.put("result", auditEvent.getAccessResult());
+ doc.put("policy", auditEvent.getPolicyId());
+ doc.put("repoType", auditEvent.getRepositoryType());
+ doc.put("resType", auditEvent.getResourceType());
+ doc.put("reason", auditEvent.getResultReason());
+ doc.put("action", auditEvent.getAction());
+ doc.put("evtTime", auditEvent.getEventTime());
+ doc.put("seq_num", auditEvent.getSeqNum());
+ doc.put("event_count", auditEvent.getEventCount());
+ doc.put("event_dur_ms", auditEvent.getEventDurationMS());
+ doc.put("tags", auditEvent.getTags());
+ doc.put("datasets", auditEvent.getDatasets());
+ doc.put("projects", auditEvent.getProjects());
+ doc.put("cluster", auditEvent.getClusterName());
+ doc.put("zoneName", auditEvent.getZoneName());
+ doc.put("agentHost", auditEvent.getAgentHostname());
+ doc.put("policyVersion", auditEvent.getPolicyVersion());
+
+ return doc;
+ }
+
+ private String connectionString() {
+ return String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, hosts, port, index);
}
private RestHighLevelClient newClient() {
@@ -261,18 +318,22 @@ public class ElasticSearchAuditDestination extends AuditDestination {
if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) {
subject = CredentialsProviderUtil.login(user, password);
}
- RestClientBuilder restClientBuilder =
- getRestClientBuilder(hosts, protocol, user, password, port);
+
+ RestClientBuilder restClientBuilder = getRestClientBuilder(hosts, protocol, user, password, port);
+
try (RestHighLevelClient restHighLevelClient = new RestHighLevelClient(restClientBuilder)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Initialized client");
}
+
boolean exists = false;
+
try {
exists = restHighLevelClient.indices().open(new OpenIndexRequest(this.index), RequestOptions.DEFAULT).isShardsAcknowledged();
} catch (Exception e) {
- LOG.warn("Error validating index " + this.index);
+ LOG.warn("Error validating index {}", this.index);
}
+
if (exists) {
if (LOG.isDebugEnabled()) {
LOG.debug("Index exists");
@@ -280,14 +341,17 @@ public class ElasticSearchAuditDestination extends AuditDestination {
} else {
LOG.info("Index does not exist");
}
+
return restHighLevelClient;
}
} catch (Throwable t) {
lastLoggedAt.updateAndGet(lastLoggedAt -> {
- long now = System.currentTimeMillis();
+ long now = System.currentTimeMillis();
long elapsed = now - lastLoggedAt;
+
if (elapsed > TimeUnit.MINUTES.toMillis(1)) {
- LOG.error("Can't connect to ElasticSearch server: " + connectionString(), t);
+ LOG.error("Can't connect to ElasticSearch server: {}", connectionString(), t);
+
return now;
} else {
return lastLoggedAt;
@@ -299,54 +363,25 @@ public class ElasticSearchAuditDestination extends AuditDestination {
private String getHosts() {
String urls = MiscUtil.getStringProperty(props, propPrefix + "." + CONFIG_URLS);
+
if (urls != null) {
urls = urls.trim();
}
+
if ("NONE".equalsIgnoreCase(urls)) {
urls = null;
}
+
return urls;
}
private String getStringProperty(Properties props, String propName, String defaultValue) {
String value = MiscUtil.getStringProperty(props, propName);
+
if (null == value) {
return defaultValue;
}
- return value;
- }
- Map<String, Object> toDoc(AuthzAuditEvent auditEvent) {
- Map<String, Object> doc = new HashMap<>();
- doc.put("id", auditEvent.getEventId());
- doc.put("access", auditEvent.getAccessType());
- doc.put("enforcer", auditEvent.getAclEnforcer());
- doc.put("agent", auditEvent.getAgentId());
- doc.put("repo", auditEvent.getRepositoryName());
- doc.put("sess", auditEvent.getSessionId());
- doc.put("reqUser", auditEvent.getUser());
- doc.put("reqData", auditEvent.getRequestData());
- doc.put("resource", auditEvent.getResourcePath());
- doc.put("cliIP", auditEvent.getClientIP());
- doc.put("logType", auditEvent.getLogType());
- doc.put("result", auditEvent.getAccessResult());
- doc.put("policy", auditEvent.getPolicyId());
- doc.put("repoType", auditEvent.getRepositoryType());
- doc.put("resType", auditEvent.getResourceType());
- doc.put("reason", auditEvent.getResultReason());
- doc.put("action", auditEvent.getAction());
- doc.put("evtTime", auditEvent.getEventTime());
- doc.put("seq_num", auditEvent.getSeqNum());
- doc.put("event_count", auditEvent.getEventCount());
- doc.put("event_dur_ms", auditEvent.getEventDurationMS());
- doc.put("tags", auditEvent.getTags());
- doc.put("datasets", auditEvent.getDatasets());
- doc.put("projects", auditEvent.getProjects());
- doc.put("cluster", auditEvent.getClusterName());
- doc.put("zoneName", auditEvent.getZoneName());
- doc.put("agentHost", auditEvent.getAgentHostname());
- doc.put("policyVersion", auditEvent.getPolicyVersion());
- return doc;
+ return value;
}
-
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
index 2bab08ac2..7b7623d03 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
@@ -19,6 +19,11 @@
package org.apache.ranger.audit.destination;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
@@ -29,216 +34,234 @@ import java.util.Date;
import java.util.List;
import java.util.Properties;
-import org.apache.ranger.audit.model.AuditEventBase;
-import org.apache.ranger.audit.provider.MiscUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* This class write the logs to local file
*/
public class FileAuditDestination extends AuditDestination {
- private static final Logger logger = LoggerFactory
- .getLogger(FileAuditDestination.class);
-
- public static final String PROP_FILE_LOCAL_DIR = "dir";
- public static final String PROP_FILE_LOCAL_FILE_NAME_FORMAT = "filename.format";
- public static final String PROP_FILE_FILE_ROLLOVER = "file.rollover.sec";
-
- String baseFolder = null;
- String fileFormat = null;
- int fileRolloverSec = 24 * 60 * 60; // In seconds
- private String logFileNameFormat;
-
- boolean initDone = false;
-
- private File logFolder;
- PrintWriter logWriter = null;
-
- private Date fileCreateTime = null;
-
- private String currentFileName;
-
- private boolean isStopped = false;
-
- @Override
- public void init(Properties prop, String propPrefix) {
- super.init(prop, propPrefix);
-
- // Initialize properties for this class
- // Initial folder and file properties
- String logFolderProp = MiscUtil.getStringProperty(props, propPrefix
- + "." + PROP_FILE_LOCAL_DIR);
- logFileNameFormat = MiscUtil.getStringProperty(props, propPrefix + "."
- + PROP_FILE_LOCAL_FILE_NAME_FORMAT);
- fileRolloverSec = MiscUtil.getIntProperty(props, propPrefix + "."
- + PROP_FILE_FILE_ROLLOVER, fileRolloverSec);
-
- if (logFolderProp == null || logFolderProp.isEmpty()) {
- logger.error("File destination folder is not configured. Please set {}. {}. name= {}", propPrefix, PROP_FILE_LOCAL_DIR, getName());
- return;
- }
- logFolder = new File(logFolderProp);
- if (!logFolder.isDirectory()) {
- logFolder.mkdirs();
- if (!logFolder.isDirectory()) {
- logger.error("FileDestination folder not found and can't be created. folder={}, name={}", logFolder.getAbsolutePath(), getName());
- return;
- }
- }
- logger.info("logFolder={}, name={}", logFolder, getName());
-
- if (logFileNameFormat == null || logFileNameFormat.isEmpty()) {
- logFileNameFormat = "%app-type%_ranger_audit.log";
- }
-
- logger.info("logFileNameFormat={}, destName={}", logFileNameFormat, getName());
-
- initDone = true;
- }
-
- @Override
- synchronized public boolean logJSON(Collection<String> events) {
- logStatusIfRequired();
- addTotalCount(events.size());
-
- if (isStopped) {
- logError("logJSON() called after stop was requested. name={}", getName());
- addDeferredCount(events.size());
- return false;
- }
-
- try {
- PrintWriter out = getLogFileStream();
- for (String event : events) {
- out.println(event);
- }
- out.flush();
- } catch (Throwable t) {
- addDeferredCount(events.size());
- logError("Error writing to log file.", t);
- return false;
- }
- addSuccessCount(events.size());
- return true;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#log(java.util.Collection)
- */
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- if (isStopped) {
- addTotalCount(events.size());
- addDeferredCount(events.size());
- logError("log() called after stop was requested. name={}", getName());
- return false;
- }
- List<String> jsonList = new ArrayList<String>();
- for (AuditEventBase event : events) {
- try {
- jsonList.add(MiscUtil.stringify(event));
- } catch (Throwable t) {
- addTotalCount(1);
- addFailedCount(1);
- logFailedEvent(event);
- logger.error("Error converting to JSON. event={}", event);
- }
- }
- return logJSON(jsonList);
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#start()
- */
- @Override
- public void start() {
- // Nothing to do here. We will open the file when the first log request
- // comes
- }
-
- @Override
- synchronized public void stop() {
- isStopped = true;
- if (logWriter != null) {
- try {
- logWriter.flush();
- logWriter.close();
- } catch (Throwable t) {
- logger.error("Error on closing log writer. Exception will be ignored. name= {}, fileName= {}", getName(), currentFileName);
- }
- logWriter = null;
- }
- logStatus();
- }
-
- // Helper methods in this class
- synchronized private PrintWriter getLogFileStream() throws Exception {
- closeFileIfNeeded();
-
- // Either there are no open log file or the previous one has been rolled
- // over
- if (logWriter == null) {
- Date currentTime = new Date();
- // Create a new file
- String fileName = MiscUtil.replaceTokens(logFileNameFormat,
- currentTime.getTime());
- File outLogFile = new File(logFolder, fileName);
- if (outLogFile.exists()) {
- // Let's try to get the next available file
- int i = 0;
- while (true) {
- i++;
- int lastDot = fileName.lastIndexOf('.');
- String baseName = fileName.substring(0, lastDot);
- String extension = fileName.substring(lastDot);
- String newFileName = baseName + "." + i + extension;
- File newLogFile = new File(logFolder, newFileName);
- if (!newLogFile.exists()) {
- // Move the file
- if (!outLogFile.renameTo(newLogFile)) {
- logger.error("Error renameing file. {} to {} " , outLogFile, newLogFile);
- }
- break;
- }
- }
- }
- if (!outLogFile.exists()) {
- logger.info("Creating new file. destName={} , fileName={} ", getName(), fileName);
- // Open the file
- logWriter = new PrintWriter(new BufferedWriter(new FileWriter(
- outLogFile)));
- } else {
- logWriter = new PrintWriter(new BufferedWriter(new FileWriter(
- outLogFile, true)));
- }
- fileCreateTime = new Date();
- currentFileName = outLogFile.getPath();
- }
- return logWriter;
- }
-
- private void closeFileIfNeeded() {
- if (logWriter == null) {
- return;
- }
- if (System.currentTimeMillis() - fileCreateTime.getTime() > fileRolloverSec * 1000) {
- logger.info("Closing file. Rolling over. name={} , fileName={}", getName(), currentFileName);
- try {
- logWriter.flush();
- logWriter.close();
- } catch (Throwable t) {
- logger.error("Error on closing log writter. Exception will be ignored. name={} , fileName={}", getName(), currentFileName);
- }
- logWriter = null;
- currentFileName = null;
- }
- }
+ private static final Logger logger = LoggerFactory.getLogger(FileAuditDestination.class);
+
+ public static final String PROP_FILE_LOCAL_DIR = "dir";
+ public static final String PROP_FILE_LOCAL_FILE_NAME_FORMAT = "filename.format";
+ public static final String PROP_FILE_FILE_ROLLOVER = "file.rollover.sec";
+
+ String baseFolder;
+ String fileFormat;
+ int fileRolloverSec = 24 * 60 * 60; // In seconds
+ boolean initDone;
+ PrintWriter logWriter;
+
+ private String logFileNameFormat;
+ private File logFolder;
+ private Date fileCreateTime;
+ private String currentFileName;
+ private boolean isStopped;
+
+ @Override
+ public void init(Properties prop, String propPrefix) {
+ super.init(prop, propPrefix);
+
+ // Initialize properties for this class
+ // Initial folder and file properties
+ String logFolderProp = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_FILE_LOCAL_DIR);
+
+ logFileNameFormat = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_FILE_LOCAL_FILE_NAME_FORMAT);
+ fileRolloverSec = MiscUtil.getIntProperty(props, propPrefix + "." + PROP_FILE_FILE_ROLLOVER, fileRolloverSec);
+
+ if (logFolderProp == null || logFolderProp.isEmpty()) {
+ logger.error("File destination folder is not configured. Please set {}. {}. name= {}", propPrefix, PROP_FILE_LOCAL_DIR, getName());
+
+ return;
+ }
+
+ logFolder = new File(logFolderProp);
+
+ if (!logFolder.isDirectory()) {
+ logFolder.mkdirs();
+
+ if (!logFolder.isDirectory()) {
+ logger.error("FileDestination folder not found and can't be created. folder={}, name={}", logFolder.getAbsolutePath(), getName());
+
+ return;
+ }
+ }
+
+ logger.info("logFolder={}, name={}", logFolder, getName());
+
+ if (logFileNameFormat == null || logFileNameFormat.isEmpty()) {
+ logFileNameFormat = "%app-type%_ranger_audit.log";
+ }
+
+ logger.info("logFileNameFormat={}, destName={}", logFileNameFormat, getName());
+
+ initDone = true;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#start()
+ */
+ @Override
+ public void start() {
+ // Nothing to do here. We will open the file when the first log request
+ // comes
+ }
+
+ @Override
+ public synchronized void stop() {
+ isStopped = true;
+
+ if (logWriter != null) {
+ try {
+ logWriter.flush();
+ logWriter.close();
+ } catch (Throwable t) {
+ logger.error("Error on closing log writer. Exception will be ignored. name={}, fileName={}", getName(), currentFileName);
+ }
+
+ logWriter = null;
+ }
+
+ logStatus();
+ }
+
+ @Override
+ public synchronized boolean logJSON(Collection<String> events) {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ if (isStopped) {
+ logError("logJSON() called after stop was requested. name={}", getName());
+
+ addDeferredCount(events.size());
+
+ return false;
+ }
+
+ try {
+ PrintWriter out = getLogFileStream();
+
+ for (String event : events) {
+ out.println(event);
+ }
+
+ out.flush();
+ } catch (Throwable t) {
+ addDeferredCount(events.size());
+
+ logError("Error writing to log file.", t);
+
+ return false;
+ }
+
+ addSuccessCount(events.size());
+
+ return true;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#log(java.util.Collection)
+ */
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ if (isStopped) {
+ addTotalCount(events.size());
+ addDeferredCount(events.size());
+
+ logError("log() called after stop was requested. name={}", getName());
+
+ return false;
+ }
+
+ List<String> jsonList = new ArrayList<>();
+
+ for (AuditEventBase event : events) {
+ try {
+ jsonList.add(MiscUtil.stringify(event));
+ } catch (Throwable t) {
+ addTotalCount(1);
+ addFailedCount(1);
+ logFailedEvent(event);
+
+ logger.error("Error converting to JSON. event={}", event);
+ }
+ }
+
+ return logJSON(jsonList);
+ }
+
+ // Helper methods in this class
+ private synchronized PrintWriter getLogFileStream() throws Exception {
+ closeFileIfNeeded();
+
+ // Either there are no open log file or the previous one has been rolled
+ // over
+ if (logWriter == null) {
+ // Create a new file
+ Date currentTime = new Date();
+ String fileName = MiscUtil.replaceTokens(logFileNameFormat, currentTime.getTime());
+ File outLogFile = new File(logFolder, fileName);
+
+ if (outLogFile.exists()) {
+ // Let's try to get the next available file
+ int i = 0;
+
+ while (true) {
+ i++;
+
+ int lastDot = fileName.lastIndexOf('.');
+ String baseName = fileName.substring(0, lastDot);
+ String extension = fileName.substring(lastDot);
+ String newFileName = baseName + "." + i + extension;
+ File newLogFile = new File(logFolder, newFileName);
+
+ if (!newLogFile.exists()) {
+ // Move the file
+ if (!outLogFile.renameTo(newLogFile)) {
+ logger.error("Error renameing file. {} to {} ", outLogFile, newLogFile);
+ }
+
+ break;
+ }
+ }
+ }
+
+ if (!outLogFile.exists()) {
+ logger.info("Creating new file. destName={} , fileName={} ", getName(), fileName);
+
+ // Open the file
+ logWriter = new PrintWriter(new BufferedWriter(new FileWriter(outLogFile)));
+ } else {
+ logWriter = new PrintWriter(new BufferedWriter(new FileWriter(outLogFile, true)));
+ }
+
+ fileCreateTime = new Date();
+ currentFileName = outLogFile.getPath();
+ }
+
+ return logWriter;
+ }
+
+ private void closeFileIfNeeded() {
+ if (logWriter == null) {
+ return;
+ }
+
+ if (System.currentTimeMillis() - fileCreateTime.getTime() > fileRolloverSec * 1000L) {
+ logger.info("Closing file. Rolling over. name={} , fileName={}", getName(), currentFileName);
+
+ try {
+ logWriter.flush();
+ logWriter.close();
+ } catch (Throwable t) {
+ logger.error("Error on closing log writter. Exception will be ignored. name={} , fileName={}", getName(), currentFileName);
+ }
+ logWriter = null;
+ currentFileName = null;
+ }
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
index 4ad8dfd98..34aa0046d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
@@ -19,6 +19,13 @@
package org.apache.ranger.audit.destination;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.provider.AuditWriterFactory;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.utils.RangerAuditWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.File;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -27,174 +34,196 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import org.apache.ranger.audit.model.AuditEventBase;
-import org.apache.ranger.audit.provider.AuditWriterFactory;
-import org.apache.ranger.audit.provider.MiscUtil;
-import org.apache.ranger.audit.utils.RangerAuditWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* This class write the logs to local file
*/
public class HDFSAuditDestination extends AuditDestination {
- private static final Logger logger = LoggerFactory
- .getLogger(HDFSAuditDestination.class);
-
- private Map<String, String> auditConfigs = null;
- private String auditProviderName = null;
- private RangerAuditWriter auditWriter = null;
- private boolean initDone = false;
- private boolean isStopped = false;
-
- @Override
- public void init(Properties prop, String propPrefix) {
- super.init(prop, propPrefix);
- this.auditProviderName = getName();
- this.auditConfigs = configProps;
-
- try {
- this.auditWriter = getWriter();
- this.initDone = true;
- } catch (Exception e) {
- logger.error("Error while getting Audit writer", e);
- }
- }
-
- @Override
- synchronized public boolean logJSON(final Collection<String> events) {
- logStatusIfRequired();
- addTotalCount(events.size());
-
- if (!initDone) {
- addDeferredCount(events.size());
- return false;
- }
- if (isStopped) {
- addDeferredCount(events.size());
- logError("log() called after stop was requested. name={}", getName());
- return false;
- }
- try {
- boolean ret = auditWriter.log(events);
- if (!ret) {
- addDeferredCount(events.size());
- return false;
- }
- } catch (Throwable t) {
- addDeferredCount(events.size());
- logError("Error writing to log file.", t);
- return false;
- } finally {
- if (logger.isDebugEnabled()) {
- logger.debug("Flushing HDFS audit. Event Size:{}", events.size());
- }
- if (auditWriter != null) {
- flush();
- }
- }
- addSuccessCount(events.size());
- return true;
- }
-
- @Override
- synchronized public boolean logFile(final File file) {
- logStatusIfRequired();
- if (!initDone) {
- return false;
- }
- if (isStopped) {
- logError("log() called after stop was requested. name={}", getName());
- return false;
- }
-
- try {
- boolean ret = auditWriter.logFile(file);
- if (!ret) {
- return false;
- }
- } catch (Throwable t) {
- logError("Error writing to log file.", t);
- return false;
- } finally {
- logger.info("Flushing HDFS audit. File:{}{}", file.getAbsolutePath(), file.getName());
- if (auditWriter != null) {
- flush();
- }
- }
- return true;
- }
-
- @Override
- public void flush() {
- if (logger.isDebugEnabled()) {
- logger.debug("==> HDFSAuditDestination.flush() called. name={}", getName());
- }
- try {
- MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<Void>) () -> {
- auditWriter.flush();
- return null;
- });
- } catch (Exception excp) {
- logger.error("HDFSAuditDestination.flush() failed", excp);
- }
-
- if (logger.isDebugEnabled()) {
- logger.debug("<== HDFSAuditDestination.flush() called. name={}", getName());
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#log(java.util.Collection)
- */
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- if (isStopped) {
- logStatusIfRequired();
- addTotalCount(events.size());
- addDeferredCount(events.size());
- logError("log() called after stop was requested. name={}", getName());
- return false;
- }
- List<String> jsonList = new ArrayList<String>();
- for (AuditEventBase event : events) {
- try {
- jsonList.add(MiscUtil.stringify(event));
- } catch (Throwable t) {
- logger.error("Error converting to JSON. event={}", event);
- addTotalCount(1);
- addFailedCount(1);
- logFailedEvent(event);
- }
- }
- return logJSON(jsonList);
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#start()
- */
- @Override
- public void start() {
- // Nothing to do here. We will open the file when the first log request
- // comes
- }
-
- @Override
- synchronized public void stop() {
- auditWriter.stop();
- logStatus();
- isStopped = true;
- }
-
- public RangerAuditWriter getWriter() throws Exception {
- AuditWriterFactory auditWriterFactory = AuditWriterFactory.getInstance();
- auditWriterFactory.init(props, propPrefix, auditProviderName, auditConfigs);
- return auditWriterFactory.getAuditWriter();
- }
+ private static final Logger logger = LoggerFactory.getLogger(HDFSAuditDestination.class);
+
+ private Map<String, String> auditConfigs;
+ private String auditProviderName;
+ private RangerAuditWriter auditWriter;
+ private boolean initDone;
+ private boolean isStopped;
+
+ @Override
+ public void init(Properties prop, String propPrefix) {
+ super.init(prop, propPrefix);
+
+ this.auditProviderName = getName();
+ this.auditConfigs = configProps;
+
+ try {
+ this.auditWriter = getWriter();
+ this.initDone = true;
+ } catch (Exception e) {
+ logger.error("Error while getting Audit writer", e);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#start()
+ */
+ @Override
+ public void start() {
+ // Nothing to do here. We will open the file when the first log request
+ // comes
+ }
+
+ @Override
+ public synchronized void stop() {
+ auditWriter.stop();
+
+ logStatus();
+
+ isStopped = true;
+ }
+
+ @Override
+ public void flush() {
+ if (logger.isDebugEnabled()) {
+ logger.debug("==> HDFSAuditDestination.flush() called. name={}", getName());
+ }
+
+ try {
+ MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<Void>) () -> {
+ auditWriter.flush();
+ return null;
+ });
+ } catch (Exception excp) {
+ logger.error("HDFSAuditDestination.flush() failed", excp);
+ }
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("<== HDFSAuditDestination.flush() called. name={}", getName());
+ }
+ }
+
+ @Override
+ public synchronized boolean logJSON(final Collection<String> events) {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ if (!initDone) {
+ addDeferredCount(events.size());
+
+ return false;
+ }
+
+ if (isStopped) {
+ addDeferredCount(events.size());
+
+ logError("log() called after stop was requested. name={}", getName());
+
+ return false;
+ }
+
+ try {
+ boolean ret = auditWriter.log(events);
+
+ if (!ret) {
+ addDeferredCount(events.size());
+
+ return false;
+ }
+ } catch (Throwable t) {
+ addDeferredCount(events.size());
+
+ logError("Error writing to log file.", t);
+
+ return false;
+ } finally {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Flushing HDFS audit. Event Size:{}", events.size());
+ }
+
+ if (auditWriter != null) {
+ flush();
+ }
+ }
+
+ addSuccessCount(events.size());
+
+ return true;
+ }
+
+ @Override
+ public synchronized boolean logFile(final File file) {
+ logStatusIfRequired();
+
+ if (!initDone) {
+ return false;
+ }
+
+ if (isStopped) {
+ logError("log() called after stop was requested. name={}", getName());
+
+ return false;
+ }
+
+ try {
+ boolean ret = auditWriter.logFile(file);
+
+ if (!ret) {
+ return false;
+ }
+ } catch (Throwable t) {
+ logError("Error writing to log file.", t);
+
+ return false;
+ } finally {
+ logger.info("Flushing HDFS audit. File:{}{}", file.getAbsolutePath(), file.getName());
+
+ if (auditWriter != null) {
+ flush();
+ }
+ }
+
+ return true;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#log(java.util.Collection)
+ */
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ if (isStopped) {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+ addDeferredCount(events.size());
+
+ logError("log() called after stop was requested. name={}", getName());
+
+ return false;
+ }
+
+ List<String> jsonList = new ArrayList<>();
+
+ for (AuditEventBase event : events) {
+ try {
+ jsonList.add(MiscUtil.stringify(event));
+ } catch (Throwable t) {
+ logger.error("Error converting to JSON. event={}", event);
+
+ addTotalCount(1);
+ addFailedCount(1);
+ logFailedEvent(event);
+ }
+ }
+
+ return logJSON(jsonList);
+ }
+
+ public RangerAuditWriter getWriter() throws Exception {
+ AuditWriterFactory auditWriterFactory = AuditWriterFactory.getInstance();
+
+ auditWriterFactory.init(props, propPrefix, auditProviderName, auditConfigs);
+
+ return auditWriterFactory.getAuditWriter();
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
index 6da3e7587..8e1c20a8c 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
@@ -19,109 +19,115 @@
package org.apache.ranger.audit.destination;
-import java.util.Collection;
-import java.util.Properties;
-
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.provider.MiscUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+import java.util.Properties;
+
public class Log4JAuditDestination extends AuditDestination {
- private static final Logger logger = LoggerFactory
- .getLogger(Log4JAuditDestination.class);
-
- private static Logger auditLogger = null;
-
- public static final String PROP_LOG4J_LOGGER = "logger";
- public static final String DEFAULT_LOGGER_PREFIX = "ranger.audit";
- private String loggerName = null;
-
- public Log4JAuditDestination() {
- logger.info("Log4JAuditDestination() called.");
-
- }
-
- @Override
- public void init(Properties prop, String propPrefix) {
- super.init(prop, propPrefix);
- loggerName = MiscUtil.getStringProperty(props, propPrefix + "."
- + PROP_LOG4J_LOGGER);
- if (loggerName == null || loggerName.isEmpty()) {
- loggerName = DEFAULT_LOGGER_PREFIX + "." + getName();
- logger.info("Logger property " + propPrefix + "."
- + PROP_LOG4J_LOGGER + " was not set. Constructing default="
- + loggerName);
- }
- logger.info("Logger name for " + getName() + " is " + loggerName);
- auditLogger = LoggerFactory.getLogger(loggerName);
- logger.info("Done initializing logger for audit. name=" + getName()
- + ", loggerName=" + loggerName);
- }
-
-
- @Override
- public void stop() {
- super.stop();
- logStatus();
- }
-
- @Override
- public boolean log(AuditEventBase event) {
- if (!auditLogger.isInfoEnabled()) {
- logStatusIfRequired();
- addTotalCount(1);
- return true;
- }
-
- if (event != null) {
- String eventStr = MiscUtil.stringify(event);
- logJSON(eventStr);
- }
- return true;
- }
-
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- if (!auditLogger.isInfoEnabled()) {
- logStatusIfRequired();
- addTotalCount(events.size());
- return true;
- }
-
- for (AuditEventBase event : events) {
- log(event);
- }
- return true;
- }
-
- @Override
- public boolean logJSON(String event) {
- logStatusIfRequired();
- addTotalCount(1);
- if (!auditLogger.isInfoEnabled()) {
- return true;
- }
-
- if (event != null) {
- auditLogger.info(event);
- addSuccessCount(1);
- }
- return true;
- }
-
- @Override
- public boolean logJSON(Collection<String> events) {
- if (!auditLogger.isInfoEnabled()) {
- logStatusIfRequired();
- addTotalCount(events.size());
- return true;
- }
-
- for (String event : events) {
- logJSON(event);
- }
- return false;
- }
+ private static final Logger logger = LoggerFactory.getLogger(Log4JAuditDestination.class);
+ private static Logger auditLogger;
+
+ public static final String PROP_LOG4J_LOGGER = "logger";
+ public static final String DEFAULT_LOGGER_PREFIX = "ranger.audit";
+
+ public Log4JAuditDestination() {
+ logger.info("Log4JAuditDestination() called.");
+ }
+
+ @Override
+ public void init(Properties prop, String propPrefix) {
+ super.init(prop, propPrefix);
+
+ String loggerName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG4J_LOGGER);
+
+ if (loggerName == null || loggerName.isEmpty()) {
+ loggerName = DEFAULT_LOGGER_PREFIX + "." + getName();
+
+ logger.info("Logger property {}.{} was not set. Constructing default={}", propPrefix, PROP_LOG4J_LOGGER, loggerName);
+ }
+
+ logger.info("Logger name for {} is {}", getName(), loggerName);
+
+ auditLogger = LoggerFactory.getLogger(loggerName);
+
+ logger.info("Done initializing logger for audit. name={}, loggerName={}", getName(), loggerName);
+ }
+
+ @Override
+ public void stop() {
+ super.stop();
+
+ logStatus();
+ }
+
+ @Override
+ public boolean log(AuditEventBase event) {
+ if (!auditLogger.isInfoEnabled()) {
+ logStatusIfRequired();
+ addTotalCount(1);
+
+ return true;
+ }
+
+ if (event != null) {
+ String eventStr = MiscUtil.stringify(event);
+
+ logJSON(eventStr);
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean logJSON(String event) {
+ logStatusIfRequired();
+ addTotalCount(1);
+
+ if (!auditLogger.isInfoEnabled()) {
+ return true;
+ }
+
+ if (event != null) {
+ auditLogger.info(event);
+ addSuccessCount(1);
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean logJSON(Collection<String> events) {
+ if (!auditLogger.isInfoEnabled()) {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ return true;
+ }
+
+ for (String event : events) {
+ logJSON(event);
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ if (!auditLogger.isInfoEnabled()) {
+ logStatusIfRequired();
+ addTotalCount(events.size());
+
+ return true;
+ }
+
+ for (AuditEventBase event : events) {
+ log(event);
+ }
+ return true;
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index 017e15b2e..b0003bfae 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -25,26 +25,32 @@ import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.audit.utils.InMemoryJAASConfiguration;
import org.apache.ranger.audit.utils.KerberosAction;
-import org.apache.ranger.audit.utils.KerberosUser;
import org.apache.ranger.audit.utils.KerberosJAASConfigUser;
+import org.apache.ranger.audit.utils.KerberosUser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
-import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.security.auth.login.LoginException;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Field;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
@@ -54,443 +60,458 @@ import java.security.SecureRandom;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
+import java.util.Optional;
import java.util.Properties;
-import javax.net.ssl.KeyManager;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.security.auth.login.LoginException;
+public class SolrAuditDestination extends AuditDestination {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrAuditDestination.class);
+
+ public static final String PROP_SOLR_URLS = "urls";
+ public static final String PROP_SOLR_ZK = "zookeepers";
+ public static final String PROP_SOLR_COLLECTION = "collection";
+ public static final String PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG = "force.use.inmemory.jaas.config";
+ public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
+ public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
+
+ private volatile SolrClient solrClient;
+ private volatile KerberosUser kerberosUser;
+
+ public SolrAuditDestination() {
+ }
+
+ @Override
+ public void init(Properties props, String propPrefix) {
+ LOG.info("init() called");
+
+ super.init(props, propPrefix);
+
+ init();
+ connect();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("SolrAuditDestination.stop() called..");
+
+ logStatus();
+
+ if (solrClient != null) {
+ try {
+ solrClient.close();
+ } catch (IOException ioe) {
+ LOG.error("Error while stopping solr!", ioe);
+ } finally {
+ solrClient = null;
+ }
+ }
+
+ if (kerberosUser != null) {
+ try {
+ kerberosUser.logout();
+ } catch (LoginException excp) {
+ LOG.error("Error logging out keytab user", excp);
+ } finally {
+ kerberosUser = null;
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ }
-import java.util.Arrays;
-import java.util.Optional;
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ boolean ret = false;
+ try {
+ logStatusIfRequired();
+ addTotalCount(events.size());
-public class SolrAuditDestination extends AuditDestination {
- private static final Logger LOG = LoggerFactory
- .getLogger(SolrAuditDestination.class);
-
- public static final String PROP_SOLR_URLS = "urls";
- public static final String PROP_SOLR_ZK = "zookeepers";
- public static final String PROP_SOLR_COLLECTION = "collection";
- public static final String PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG = "force.use.inmemory.jaas.config";
-
- public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
- public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
-
- private volatile SolrClient solrClient = null;
- private volatile KerberosUser kerberosUser = null;
-
- public SolrAuditDestination() {
- }
-
- @Override
- public void init(Properties props, String propPrefix) {
- LOG.info("init() called");
- super.init(props, propPrefix);
- init();
- connect();
- }
-
- @Override
- public void stop() {
- LOG.info("SolrAuditDestination.stop() called..");
- logStatus();
-
- if (solrClient != null) {
- try {
- solrClient.close();
- } catch (IOException ioe) {
- LOG.error("Error while stopping slor!", ioe);
- } finally {
- solrClient = null;
- }
- }
-
- if (kerberosUser != null) {
- try {
- kerberosUser.logout();
- } catch (LoginException excp) {
- LOG.error("Error logging out keytab user", excp);
- } finally {
- kerberosUser = null;
- }
- }
- }
-
- synchronized void connect() {
- SolrClient me = solrClient;
- if (me == null) {
- synchronized(SolrAuditDestination.class) {
- me = solrClient;
- if (solrClient == null) {
- KeyManager[] kmList = getKeyManagers();
- TrustManager[] tmList = getTrustManagers();
- SSLContext sslContext = getSSLContext(kmList, tmList);
- if(sslContext != null) {
- SSLContext.setDefault(sslContext);
- }
- String urls = MiscUtil.getStringProperty(props, propPrefix
- + "." + PROP_SOLR_URLS);
- if (urls != null) {
- urls = urls.trim();
- }
- if (urls != null && urls.equalsIgnoreCase("NONE")) {
- urls = null;
- }
- List<String> solrURLs = new ArrayList<String>();
- String zkHosts = null;
- solrURLs = MiscUtil.toArray(urls, ",");
- zkHosts = MiscUtil.getStringProperty(props, propPrefix + "."
- + PROP_SOLR_ZK);
- if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) {
- zkHosts = null;
- }
- String collectionName = MiscUtil.getStringProperty(props,
- propPrefix + "." + PROP_SOLR_COLLECTION);
- if (collectionName == null
- || collectionName.equalsIgnoreCase("none")) {
- collectionName = DEFAULT_COLLECTION_NAME;
- }
-
- LOG.info("Solr zkHosts=" + zkHosts + ", solrURLs=" + urls
- + ", collectionName=" + collectionName);
-
- if (zkHosts != null && !zkHosts.isEmpty()) {
- LOG.info("Connecting to solr cloud using zkHosts="
- + zkHosts);
- try (Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder()) {
- SolrHttpClientBuilder kb = krbBuild.getBuilder();
- HttpClientUtil.setHttpClientBuilder(kb);
-
- final List<String> zkhosts = new ArrayList<String>(Arrays.asList(zkHosts.split(",")));
- final CloudSolrClient solrCloudClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction<CloudSolrClient>() {
- @Override
- public CloudSolrClient run() throws Exception {
- CloudSolrClient solrCloudClient = new CloudSolrClient.Builder(zkhosts, Optional.empty()).build();
- return solrCloudClient;
- }
-
- ;
- });
-
- solrCloudClient.setDefaultCollection(collectionName);
- me = solrClient = solrCloudClient;
- } catch (Throwable t) {
- LOG.error("Can't connect to Solr server. ZooKeepers="
- + zkHosts, t);
- }
- } else if (solrURLs != null && !solrURLs.isEmpty()) {
- try (Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder()) {
- LOG.info("Connecting to Solr using URLs=" + solrURLs);
- SolrHttpClientBuilder kb = krbBuild.getBuilder();
- HttpClientUtil.setHttpClientBuilder(kb);
- final List<String> solrUrls = solrURLs;
- final LBHttpSolrClient lbSolrClient = MiscUtil.executePrivilegedAction(new PrivilegedExceptionAction<LBHttpSolrClient>() {
- @Override
- public LBHttpSolrClient run() throws Exception {
- LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
- builder.withBaseSolrUrl(solrUrls.get(0));
- builder.withConnectionTimeout(1000);
- LBHttpSolrClient lbSolrClient = builder.build();
- return lbSolrClient;
- }
-
- ;
- });
-
- for (int i = 1; i < solrURLs.size(); i++) {
- lbSolrClient.addSolrServer(solrURLs.get(i));
- }
- me = solrClient = lbSolrClient;
- } catch (Throwable t) {
- LOG.error("Can't connect to Solr server. URL="
- + solrURLs, t);
- }
- }
- }
- }
- }
- }
-
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- boolean ret = false;
- try {
- logStatusIfRequired();
- addTotalCount(events.size());
-
- if (solrClient == null) {
- connect();
- if (solrClient == null) {
- // Solr is still not initialized. So need return error
- addDeferredCount(events.size());
- return ret;
- }
- }
-
- final Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
- for (AuditEventBase event : events) {
- AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
- // Convert AuditEventBase to Solr document
- SolrInputDocument document = toSolrDoc(authzEvent);
- docs.add(document);
- }
- try {
- final UpdateResponse response = addDocsToSolr(solrClient, docs);
-
- if (response.getStatus() != 0) {
- addFailedCount(events.size());
- logFailedEvent(events, response.toString());
- } else {
- addSuccessCount(events.size());
- ret = true;
- }
- } catch (SolrException ex) {
- addFailedCount(events.size());
- logFailedEvent(events, ex);
- }
- } catch (Throwable t) {
- addDeferredCount(events.size());
- logError("Error sending message to Solr", t);
- }
- return ret;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.ranger.audit.provider.AuditProvider#flush()
- */
- @Override
- public void flush() {
-
- }
-
- SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", auditEvent.getEventId());
- doc.addField("access", auditEvent.getAccessType());
- doc.addField("enforcer", auditEvent.getAclEnforcer());
- doc.addField("agent", auditEvent.getAgentId());
- doc.addField("repo", auditEvent.getRepositoryName());
- doc.addField("sess", auditEvent.getSessionId());
- doc.addField("reqUser", auditEvent.getUser());
- doc.addField("reqData", auditEvent.getRequestData());
- doc.addField("resource", auditEvent.getResourcePath());
- doc.addField("cliIP", auditEvent.getClientIP());
- doc.addField("logType", auditEvent.getLogType());
- doc.addField("result", auditEvent.getAccessResult());
- doc.addField("policy", auditEvent.getPolicyId());
- doc.addField("repoType", auditEvent.getRepositoryType());
- doc.addField("resType", auditEvent.getResourceType());
- doc.addField("reason", auditEvent.getResultReason());
- doc.addField("action", auditEvent.getAction());
- doc.addField("evtTime", auditEvent.getEventTime());
- doc.addField("seq_num", auditEvent.getSeqNum());
- doc.setField("event_count", auditEvent.getEventCount());
- doc.setField("event_dur_ms", auditEvent.getEventDurationMS());
- doc.setField("tags", auditEvent.getTags());
- doc.addField("datasets", auditEvent.getDatasets());
- doc.addField("projects", auditEvent.getProjects());
- doc.setField("cluster", auditEvent.getClusterName());
- doc.setField("zoneName", auditEvent.getZoneName());
- doc.setField("agentHost", auditEvent.getAgentHostname());
- doc.setField("policyVersion", auditEvent.getPolicyVersion());
-
- return doc;
- }
-
- public boolean isAsync() {
- return true;
- }
-
- private void init() {
- LOG.info("==>SolrAuditDestination.init()" );
- try {
- // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
- // Acutal solrclient JAAS configs are read from the ranger-<component>-audit.xml present in components conf folder and set by InMemoryJAASConfiguration
- // Refer InMemoryJAASConfiguration doc for JAAS Configuration
- String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
- LOG.info("In solrAuditDestination.init() : JAAS Configuration set as [" + confFileName + "]");
- if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) {
- if ( MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG,false) ) {
- System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null");
- } else {
- LOG.warn("No Client JAAS config present in solr audit config. Ranger Audit to Kerberized Solr will fail...");
- }
- }
-
- LOG.info("Loading SolrClient JAAS config from Ranger audit config if present...");
-
- InMemoryJAASConfiguration conf = InMemoryJAASConfiguration.init(props);
-
- KerberosUser kerberosUser = new KerberosJAASConfigUser("Client", conf);
-
- if (kerberosUser.getPrincipal() != null) {
- this.kerberosUser = kerberosUser;
- }
- } catch (Exception e) {
- LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e);
- } finally {
- String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
- LOG.info("In solrAuditDestination.init() (finally) : JAAS Configuration set as [" + confFileName + "]");
- }
- LOG.info("<==SolrAuditDestination.init()" );
- }
-
- private KeyManager[] getKeyManagers() {
- KeyManager[] kmList = null;
- String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL);
- String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS;
- String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE);
- String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias);
- if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) {
- InputStream in = null;
-
- try {
- in = getFileInputStream(keyStoreFile);
-
- if (in != null) {
- String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE);
- keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT;
- KeyStore keyStore = KeyStore.getInstance(keyStoreType);
-
- keyStore.load(in, keyStoreFilepwd.toCharArray());
-
- KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
-
- keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
-
- kmList = keyManagerFactory.getKeyManagers();
- } else {
- LOG.error("Unable to obtain keystore from file [" + keyStoreFile + "]");
- }
- } catch (KeyStoreException e) {
- LOG.error("Unable to obtain from KeyStore :" + e.getMessage(), e);
- } catch (NoSuchAlgorithmException e) {
- LOG.error("SSL algorithm is NOT available in the environment", e);
- } catch (CertificateException e) {
- LOG.error("Unable to obtain the requested certification ", e);
- } catch (FileNotFoundException e) {
- LOG.error("Unable to find the necessary SSL Keystore Files", e);
- } catch (IOException e) {
- LOG.error("Unable to read the necessary SSL Keystore Files", e);
- } catch (UnrecoverableKeyException e) {
- LOG.error("Unable to recover the key from keystore", e);
- } finally {
- close(in, keyStoreFile);
- }
- }
-
- return kmList;
- }
-
- private TrustManager[] getTrustManagers() {
- TrustManager[] tmList = null;
- String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL);
- String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS;
- String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE);
- String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias);
- if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) {
- InputStream in = null;
-
- try {
- in = getFileInputStream(trustStoreFile);
-
- if (in != null) {
- String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE);
- trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT;
- KeyStore trustStore = KeyStore.getInstance(trustStoreType);
-
- trustStore.load(in, trustStoreFilepwd.toCharArray());
-
- TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
-
- trustManagerFactory.init(trustStore);
-
- tmList = trustManagerFactory.getTrustManagers();
- } else {
- LOG.error("Unable to obtain truststore from file [" + trustStoreFile + "]");
- }
- } catch (KeyStoreException e) {
- LOG.error("Unable to obtain from KeyStore", e);
- } catch (NoSuchAlgorithmException e) {
- LOG.error("SSL algorithm is NOT available in the environment :" + e.getMessage(), e);
- } catch (CertificateException e) {
- LOG.error("Unable to obtain the requested certification :" + e.getMessage(), e);
- } catch (FileNotFoundException e) {
- LOG.error("Unable to find the necessary SSL TrustStore File:" + trustStoreFile, e);
- } catch (IOException e) {
- LOG.error("Unable to read the necessary SSL TrustStore Files :" + trustStoreFile, e);
- } finally {
- close(in, trustStoreFile);
- }
- }
-
- return tmList;
- }
-
- private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) {
- SSLContext sslContext = null;
- try {
- sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
- if (sslContext != null) {
- sslContext.init(kmList, tmList, new SecureRandom());
- }
- } catch (NoSuchAlgorithmException e) {
- LOG.error("SSL algorithm is not available in the environment", e);
- } catch (KeyManagementException e) {
- LOG.error("Unable to initialise the SSLContext", e);
- }
- return sslContext;
- }
-
- private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection<SolrInputDocument> docs) throws Exception {
- final UpdateResponse ret;
-
- try {
- final PrivilegedExceptionAction<UpdateResponse> action = () -> solrClient.add(docs);
-
- if (kerberosUser != null) {
- // execute the privileged action as the given keytab user
- final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG);
-
- ret = (UpdateResponse) kerberosAction.execute();
- } else {
- ret = action.run();
- }
- } catch (Exception e) {
- throw e;
- }
-
- return ret;
- }
-
- private InputStream getFileInputStream(String fileName) throws IOException {
- InputStream in = null;
- if (StringUtils.isNotEmpty(fileName)) {
- File file = new File(fileName);
- if (file != null && file.exists()) {
- in = new FileInputStream(file);
- } else {
- in = ClassLoader.getSystemResourceAsStream(fileName);
- }
- }
- return in;
- }
-
- private void close(InputStream str, String filename) {
- if (str != null) {
- try {
- str.close();
- } catch (IOException excp) {
- LOG.error("Error while closing file: [" + filename + "]", excp);
- }
- }
- }
+ if (solrClient == null) {
+ connect();
+
+ if (solrClient == null) {
+ // Solr is still not initialized. So need return error
+ addDeferredCount(events.size());
+
+ return ret;
+ }
+ }
+
+ final Collection<SolrInputDocument> docs = new ArrayList<>();
+
+ for (AuditEventBase event : events) {
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+
+ // Convert AuditEventBase to Solr document
+ SolrInputDocument document = toSolrDoc(authzEvent);
+
+ docs.add(document);
+ }
+
+ try {
+ final UpdateResponse response = addDocsToSolr(solrClient, docs);
+
+ if (response.getStatus() != 0) {
+ addFailedCount(events.size());
+
+ logFailedEvent(events, response.toString());
+ } else {
+ addSuccessCount(events.size());
+
+ ret = true;
+ }
+ } catch (SolrException ex) {
+ addFailedCount(events.size());
+ logFailedEvent(events, ex);
+ }
+ } catch (Throwable t) {
+ addDeferredCount(events.size());
+
+ logError("Error sending message to Solr", t);
+ }
+
+ return ret;
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+ synchronized void connect() {
+ SolrClient me = solrClient;
+
+ if (me == null) {
+ synchronized (SolrAuditDestination.class) {
+ me = solrClient;
+
+ if (me == null) {
+ KeyManager[] kmList = getKeyManagers();
+ TrustManager[] tmList = getTrustManagers();
+ SSLContext sslContext = getSSLContext(kmList, tmList);
+
+ if (sslContext != null) {
+ SSLContext.setDefault(sslContext);
+ }
+
+ String urls = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SOLR_URLS);
+
+ if (urls != null) {
+ urls = urls.trim();
+ }
+
+ if (urls != null && urls.equalsIgnoreCase("NONE")) {
+ urls = null;
+ }
+
+ List<String> solrURLs = MiscUtil.toArray(urls, ",");
+ String zkHosts = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SOLR_ZK);
+
+ if (zkHosts != null && zkHosts.equalsIgnoreCase("NONE")) {
+ zkHosts = null;
+ }
+
+ String collectionName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_SOLR_COLLECTION);
+
+ if (collectionName == null || collectionName.equalsIgnoreCase("none")) {
+ collectionName = DEFAULT_COLLECTION_NAME;
+ }
+
+ LOG.info("Solr zkHosts={}, solrURLs={}, collectionName={}", zkHosts, urls, collectionName);
+
+ if (zkHosts != null && !zkHosts.isEmpty()) {
+ LOG.info("Connecting to solr cloud using zkHosts={}", zkHosts);
+
+ try (Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder()) {
+ SolrHttpClientBuilder kb = krbBuild.getBuilder();
+
+ HttpClientUtil.setHttpClientBuilder(kb);
+
+ final List<String> zkhosts = new ArrayList<>(Arrays.asList(zkHosts.split(",")));
+ final CloudSolrClient solrCloudClient = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<CloudSolrClient>) () -> new CloudSolrClient.Builder(zkhosts, Optional.empty()).build());
+
+ solrCloudClient.setDefaultCollection(collectionName);
+
+ me = solrCloudClient;
+ solrClient = me;
+ } catch (Throwable t) {
+ LOG.error("Can't connect to Solr server. ZooKeepers={}", zkHosts, t);
+ }
+ } else if (solrURLs != null && !solrURLs.isEmpty()) {
+ try (Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder()) {
+ LOG.info("Connecting to Solr using URLs={}", solrURLs);
+
+ SolrHttpClientBuilder kb = krbBuild.getBuilder();
+
+ HttpClientUtil.setHttpClientBuilder(kb);
+
+ final List<String> solrUrls = solrURLs;
+ final LBHttpSolrClient lbSolrClient = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<LBHttpSolrClient>) () -> {
+ LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
+
+ builder.withBaseSolrUrl(solrUrls.get(0));
+ builder.withConnectionTimeout(1000);
+
+ return builder.build();
+ });
+
+ for (int i = 1; i < solrURLs.size(); i++) {
+ lbSolrClient.addSolrServer(solrURLs.get(i));
+ }
+
+ me = lbSolrClient;
+ solrClient = me;
+ } catch (Throwable t) {
+ LOG.error("Can't connect to Solr server. URL={}", solrURLs, t);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
+ SolrInputDocument doc = new SolrInputDocument();
+
+ doc.addField("id", auditEvent.getEventId());
+ doc.addField("access", auditEvent.getAccessType());
+ doc.addField("enforcer", auditEvent.getAclEnforcer());
+ doc.addField("agent", auditEvent.getAgentId());
+ doc.addField("repo", auditEvent.getRepositoryName());
+ doc.addField("sess", auditEvent.getSessionId());
+ doc.addField("reqUser", auditEvent.getUser());
+ doc.addField("reqData", auditEvent.getRequestData());
+ doc.addField("resource", auditEvent.getResourcePath());
+ doc.addField("cliIP", auditEvent.getClientIP());
+ doc.addField("logType", auditEvent.getLogType());
+ doc.addField("result", auditEvent.getAccessResult());
+ doc.addField("policy", auditEvent.getPolicyId());
+ doc.addField("repoType", auditEvent.getRepositoryType());
+ doc.addField("resType", auditEvent.getResourceType());
+ doc.addField("reason", auditEvent.getResultReason());
+ doc.addField("action", auditEvent.getAction());
+ doc.addField("evtTime", auditEvent.getEventTime());
+ doc.addField("seq_num", auditEvent.getSeqNum());
+ doc.setField("event_count", auditEvent.getEventCount());
+ doc.setField("event_dur_ms", auditEvent.getEventDurationMS());
+ doc.setField("tags", auditEvent.getTags());
+ doc.addField("datasets", auditEvent.getDatasets());
+ doc.addField("projects", auditEvent.getProjects());
+ doc.setField("cluster", auditEvent.getClusterName());
+ doc.setField("zoneName", auditEvent.getZoneName());
+ doc.setField("agentHost", auditEvent.getAgentHostname());
+ doc.setField("policyVersion", auditEvent.getPolicyVersion());
+
+ return doc;
+ }
+
+ private void init() {
+ LOG.info("==>SolrAuditDestination.init()");
+
+ try {
+ // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
+ // Acutal solrclient JAAS configs are read from the ranger-<component>-audit.xml present in components conf folder and set by InMemoryJAASConfiguration
+ // Refer InMemoryJAASConfiguration doc for JAAS Configuration
+ String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
+
+ LOG.info("In solrAuditDestination.init() : JAAS Configuration set as [{}]", confFileName);
+
+ if (System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null) {
+ if (MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_SOLR_FORCE_USE_INMEMORY_JAAS_CONFIG, false)) {
+ System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null");
+ } else {
+ LOG.warn("No Client JAAS config present in solr audit config. Ranger Audit to Kerberized Solr will fail...");
+ }
+ }
+
+ LOG.info("Loading SolrClient JAAS config from Ranger audit config if present...");
+
+ InMemoryJAASConfiguration conf = InMemoryJAASConfiguration.init(props);
+
+ KerberosUser kerberosUser = new KerberosJAASConfigUser("Client", conf);
+
+ if (kerberosUser.getPrincipal() != null) {
+ this.kerberosUser = kerberosUser;
+ }
+ } catch (Exception e) {
+ LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e);
+ } finally {
+ String confFileName = System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG);
+ LOG.info("In solrAuditDestination.init() (finally) : JAAS Configuration set as [{}]", confFileName);
+ }
+
+ LOG.info("<==SolrAuditDestination.init()");
+ }
+
+ private KeyManager[] getKeyManagers() {
+ KeyManager[] kmList = null;
+ String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL);
+ String keyStoreAlias = RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS;
+ String keyStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE);
+ String keyStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, keyStoreAlias);
+
+ if (StringUtils.isNotEmpty(keyStoreFile) && StringUtils.isNotEmpty(keyStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(keyStoreFile);
+
+ if (in != null) {
+ String keyStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE);
+
+ keyStoreType = StringUtils.isNotEmpty(keyStoreType) ? keyStoreType : RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT;
+
+ KeyStore keyStore = KeyStore.getInstance(keyStoreType);
+
+ keyStore.load(in, keyStoreFilepwd.toCharArray());
+
+ KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(RANGER_SSL_KEYMANAGER_ALGO_TYPE);
+
+ keyManagerFactory.init(keyStore, keyStoreFilepwd.toCharArray());
+
+ kmList = keyManagerFactory.getKeyManagers();
+ } else {
+ LOG.error("Unable to obtain keystore from file [{}]", keyStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore :{}", e.getMessage(), e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment", e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification ", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL Keystore Files", e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL Keystore Files", e);
+ } catch (UnrecoverableKeyException e) {
+ LOG.error("Unable to recover the key from keystore", e);
+ } finally {
+ close(in, keyStoreFile);
+ }
+ }
+
+ return kmList;
+ }
+
+ private TrustManager[] getTrustManagers() {
+ TrustManager[] tmList = null;
+ String credentialProviderPath = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL);
+ String trustStoreAlias = RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS;
+ String trustStoreFile = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE);
+ String trustStoreFilepwd = MiscUtil.getCredentialString(credentialProviderPath, trustStoreAlias);
+
+ if (StringUtils.isNotEmpty(trustStoreFile) && StringUtils.isNotEmpty(trustStoreFilepwd)) {
+ InputStream in = null;
+
+ try {
+ in = getFileInputStream(trustStoreFile);
+
+ if (in != null) {
+ String trustStoreType = MiscUtil.getStringProperty(props, RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE);
+
+ trustStoreType = StringUtils.isNotEmpty(trustStoreType) ? trustStoreType : RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT;
+
+ KeyStore trustStore = KeyStore.getInstance(trustStoreType);
+
+ trustStore.load(in, trustStoreFilepwd.toCharArray());
+
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(RANGER_SSL_TRUSTMANAGER_ALGO_TYPE);
+
+ trustManagerFactory.init(trustStore);
+
+ tmList = trustManagerFactory.getTrustManagers();
+ } else {
+ LOG.error("Unable to obtain truststore from file [{}]", trustStoreFile);
+ }
+ } catch (KeyStoreException e) {
+ LOG.error("Unable to obtain from KeyStore", e);
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is NOT available in the environment :{}", e.getMessage(), e);
+ } catch (CertificateException e) {
+ LOG.error("Unable to obtain the requested certification :{}", e.getMessage(), e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the necessary SSL TrustStore File:{}", trustStoreFile, e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the necessary SSL TrustStore Files :{}", trustStoreFile, e);
+ } finally {
+ close(in, trustStoreFile);
+ }
+ }
+
+ return tmList;
+ }
+
+ private SSLContext getSSLContext(KeyManager[] kmList, TrustManager[] tmList) {
+ SSLContext sslContext = null;
+
+ try {
+ sslContext = SSLContext.getInstance(RANGER_SSL_CONTEXT_ALGO_TYPE);
+
+ if (sslContext != null) {
+ sslContext.init(kmList, tmList, new SecureRandom());
+ }
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("SSL algorithm is not available in the environment", e);
+ } catch (KeyManagementException e) {
+ LOG.error("Unable to initialise the SSLContext", e);
+ }
+
+ return sslContext;
+ }
+
+ private UpdateResponse addDocsToSolr(final SolrClient solrClient, final Collection<SolrInputDocument> docs) throws Exception {
+ final UpdateResponse ret;
+
+ try {
+ final PrivilegedExceptionAction<UpdateResponse> action = () -> solrClient.add(docs);
+
+ if (kerberosUser != null) {
+ // execute the privileged action as the given keytab user
+ final KerberosAction kerberosAction = new KerberosAction<>(kerberosUser, action, LOG);
+
+ ret = (UpdateResponse) kerberosAction.execute();
+ } else {
+ ret = action.run();
+ }
+ } catch (Exception e) {
+ throw e;
+ }
+
+ return ret;
+ }
+
+ private InputStream getFileInputStream(String fileName) throws IOException {
+ InputStream in = null;
+
+ if (StringUtils.isNotEmpty(fileName)) {
+ File file = new File(fileName);
+
+ if (file.exists()) {
+ in = new FileInputStream(file);
+ } else {
+ in = ClassLoader.getSystemResourceAsStream(fileName);
+ }
+ }
+
+ return in;
+ }
+
+ private void close(InputStream str, String filename) {
+ if (str != null) {
+ try {
+ str.close();
+ } catch (IOException excp) {
+ LOG.error("Error while closing file: [{}]", filename, excp);
+ }
+ }
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
index 84eaebec9..149463163 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
@@ -22,12 +22,14 @@ package org.apache.ranger.audit.model;
import java.util.Date;
public abstract class AuditEventBase {
+ protected AuditEventBase() {
+ }
- protected AuditEventBase() {
- }
+ public abstract String getEventKey();
- public abstract String getEventKey();
- public abstract Date getEventTime ();
- public abstract void setEventCount(long eventCount);
- public abstract void setEventDurationMS(long eventDurationMS);
+ public abstract Date getEventTime();
+
+ public abstract void setEventCount(long eventCount);
+
+ public abstract void setEventDurationMS(long eventDurationMS);
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditIndexRecord.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditIndexRecord.java
index d0ea2b908..1cbd664d1 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditIndexRecord.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditIndexRecord.java
@@ -22,118 +22,116 @@ package org.apache.ranger.audit.model;
import java.util.Date;
public class AuditIndexRecord {
- String id;
- String filePath;
- int linePosition = 0;
- SPOOL_FILE_STATUS status = SPOOL_FILE_STATUS.write_inprogress;
- Date fileCreateTime;
- Date writeCompleteTime;
- Date doneCompleteTime;
- Date lastSuccessTime;
- Date lastFailedTime;
- int failedAttemptCount = 0;
- boolean lastAttempt = false;
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getFilePath() {
- return filePath;
- }
-
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
-
- public int getLinePosition() {
- return linePosition;
- }
-
- public void setLinePosition(int linePosition) {
- this.linePosition = linePosition;
- }
-
- public SPOOL_FILE_STATUS getStatus() {
- return status;
- }
-
- public void setStatus(SPOOL_FILE_STATUS status) {
- this.status = status;
- }
-
- public Date getFileCreateTime() {
- return fileCreateTime;
- }
-
- public void setFileCreateTime(Date fileCreateTime) {
- this.fileCreateTime = fileCreateTime;
- }
-
- public Date getWriteCompleteTime() {
- return writeCompleteTime;
- }
-
- public void setWriteCompleteTime(Date writeCompleteTime) {
- this.writeCompleteTime = writeCompleteTime;
- }
-
- public Date getDoneCompleteTime() {
- return doneCompleteTime;
- }
-
- public void setDoneCompleteTime(Date doneCompleteTime) {
- this.doneCompleteTime = doneCompleteTime;
- }
-
- public Date getLastSuccessTime() {
- return lastSuccessTime;
- }
-
- public void setLastSuccessTime(Date lastSuccessTime) {
- this.lastSuccessTime = lastSuccessTime;
- }
-
- public Date getLastFailedTime() {
- return lastFailedTime;
- }
-
- public void setLastFailedTime(Date lastFailedTime) {
- this.lastFailedTime = lastFailedTime;
- }
-
- public int getFailedAttemptCount() {
- return failedAttemptCount;
- }
-
- public void setFailedAttemptCount(int failedAttemptCount) {
- this.failedAttemptCount = failedAttemptCount;
- }
-
- public boolean getLastAttempt() {
- return lastAttempt;
- }
-
- public void setLastAttempt(boolean lastAttempt) {
- this.lastAttempt = lastAttempt;
- }
-
- @Override
- public String toString() {
- return "AuditIndexRecord [id=" + id + ", filePath=" + filePath
- + ", linePosition=" + linePosition + ", status=" + status
- + ", fileCreateTime=" + fileCreateTime
- + ", writeCompleteTime=" + writeCompleteTime
- + ", doneCompleteTime=" + doneCompleteTime
- + ", lastSuccessTime=" + lastSuccessTime
- + ", lastFailedTime=" + lastFailedTime
- + ", failedAttemptCount=" + failedAttemptCount
- + ", lastAttempt=" + lastAttempt + "]";
- }
+ String id;
+ String filePath;
+ int linePosition;
+ SPOOL_FILE_STATUS status = SPOOL_FILE_STATUS.write_inprogress;
+ Date fileCreateTime;
+ Date writeCompleteTime;
+ Date doneCompleteTime;
+ Date lastSuccessTime;
+ Date lastFailedTime;
+ int failedAttemptCount;
+ boolean lastAttempt;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public int getLinePosition() {
+ return linePosition;
+ }
+
+ public void setLinePosition(int linePosition) {
+ this.linePosition = linePosition;
+ }
+
+ public SPOOL_FILE_STATUS getStatus() {
+ return status;
+ }
+
+ public void setStatus(SPOOL_FILE_STATUS status) {
+ this.status = status;
+ }
+
+ public Date getFileCreateTime() {
+ return fileCreateTime;
+ }
+
+ public void setFileCreateTime(Date fileCreateTime) {
+ this.fileCreateTime = fileCreateTime;
+ }
+
+ public Date getWriteCompleteTime() {
+ return writeCompleteTime;
+ }
+
+ public void setWriteCompleteTime(Date writeCompleteTime) {
+ this.writeCompleteTime = writeCompleteTime;
+ }
+
+ public Date getDoneCompleteTime() {
+ return doneCompleteTime;
+ }
+
+ public void setDoneCompleteTime(Date doneCompleteTime) {
+ this.doneCompleteTime = doneCompleteTime;
+ }
+
+ public Date getLastSuccessTime() {
+ return lastSuccessTime;
+ }
+
+ public void setLastSuccessTime(Date lastSuccessTime) {
+ this.lastSuccessTime = lastSuccessTime;
+ }
+
+ public Date getLastFailedTime() {
+ return lastFailedTime;
+ }
+
+ public void setLastFailedTime(Date lastFailedTime) {
+ this.lastFailedTime = lastFailedTime;
+ }
+
+ public int getFailedAttemptCount() {
+ return failedAttemptCount;
+ }
+
+ public void setFailedAttemptCount(int failedAttemptCount) {
+ this.failedAttemptCount = failedAttemptCount;
+ }
+
+ public boolean getLastAttempt() {
+ return lastAttempt;
+ }
+
+ public void setLastAttempt(boolean lastAttempt) {
+ this.lastAttempt = lastAttempt;
+ }
+
+ @Override
+ public String toString() {
+ return "AuditIndexRecord [id=" + id + ", filePath=" + filePath
+ + ", linePosition=" + linePosition + ", status=" + status
+ + ", fileCreateTime=" + fileCreateTime
+ + ", writeCompleteTime=" + writeCompleteTime
+ + ", doneCompleteTime=" + doneCompleteTime
+ + ", lastSuccessTime=" + lastSuccessTime
+ + ", lastFailedTime=" + lastFailedTime
+ + ", failedAttemptCount=" + failedAttemptCount
+ + ", lastAttempt=" + lastAttempt + "]";
+ }
}
-
-
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
index 5ed88eb6a..e6c3f2f11 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
@@ -19,582 +19,566 @@
package org.apache.ranger.audit.model;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Set;
-
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.commons.lang.StringUtils;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
@JsonSerialize
public class AuthzAuditEvent extends AuditEventBase {
- protected static String FIELD_SEPARATOR = ";";
-
- protected static final int MAX_ACTION_FIELD_SIZE = 1800;
- protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800;
+ protected static final int MAX_ACTION_FIELD_SIZE = 1800;
+ protected static final int MAX_REQUEST_DATA_FIELD_SIZE = 1800;
+ protected static String FIELD_SEPARATOR = ";";
- @JsonProperty("repoType")
- protected int repositoryType = 0;
+ @JsonProperty("repoType")
+ protected int repositoryType;
- @JsonProperty("repo")
- protected String repositoryName = null;
+ @JsonProperty("repo")
+ protected String repositoryName;
- @JsonProperty("reqUser")
- protected String user = null;
+ @JsonProperty("reqUser")
+ protected String user;
- @JsonProperty("evtTime")
- protected Date eventTime = new Date();
+ @JsonProperty("evtTime")
+ protected Date eventTime = new Date();
- @JsonProperty("access")
- protected String accessType = null;
+ @JsonProperty("access")
+ protected String accessType;
- @JsonProperty("resource")
- protected String resourcePath = null;
+ @JsonProperty("resource")
+ protected String resourcePath;
- @JsonProperty("resType")
- protected String resourceType = null;
+ @JsonProperty("resType")
+ protected String resourceType;
- @JsonProperty("action")
- protected String action = null;
+ @JsonProperty("action")
+ protected String action;
- @JsonProperty("result")
- protected short accessResult = 0; // 0 - DENIED; 1 - ALLOWED; HTTP return
- // code
+ @JsonProperty("result")
+ protected short accessResult; // 0 - DENIED; 1 - ALLOWED; HTTP return code
- @JsonProperty("agent")
- protected String agentId = null;
+ @JsonProperty("agent")
+ protected String agentId;
- @JsonProperty("policy")
- protected long policyId = 0;
+ @JsonProperty("policy")
+ protected long policyId;
- @JsonProperty("reason")
- protected String resultReason = null;
+ @JsonProperty("reason")
+ protected String resultReason;
- @JsonProperty("enforcer")
- protected String aclEnforcer = null;
+ @JsonProperty("enforcer")
+ protected String aclEnforcer;
- @JsonProperty("sess")
- protected String sessionId = null;
-
- @JsonProperty("cliType")
- protected String clientType = null;
-
- @JsonProperty("cliIP")
- protected String clientIP = null;
-
- @JsonProperty("reqData")
- protected String requestData = null;
-
- @JsonProperty("agentHost")
- protected String agentHostname = null;
-
- @JsonProperty("logType")
- protected String logType = null;
-
- @JsonProperty("id")
- protected String eventId = null;
-
- /**
- * This to ensure order within a session. Order not guaranteed across
- * processes and hosts
- */
- @JsonProperty("seq_num")
- protected long seqNum = 0;
-
- @JsonProperty("event_count")
- protected long eventCount = 1;
-
- @JsonProperty("event_dur_ms")
- protected long eventDurationMS = 0;
-
- @JsonProperty("tags")
- protected Set<String> tags = new HashSet<>();
-
- @JsonProperty("datasets")
- protected Set<String> datasets = null;
-
- @JsonProperty("projects")
- protected Set<String> projects = null;
-
- @JsonProperty("additional_info")
- protected String additionalInfo;
-
- @JsonProperty("cluster_name")
- protected String clusterName;
-
- @JsonProperty("zone_name")
- protected String zoneName;
-
- @JsonProperty("policy_version")
- protected Long policyVersion;
-
- public AuthzAuditEvent() {
- super();
-
- this.repositoryType = 0;
- }
-
- public AuthzAuditEvent(int repositoryType, String repositoryName,
- String user, Date eventTime, String accessType,
- String resourcePath, String resourceType, String action,
- short accessResult, String agentId, long policyId,
- String resultReason, String aclEnforcer, String sessionId,
- String clientType, String clientIP, String requestData, String clusterName) {
- this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
- policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, null);
- }
-
- public AuthzAuditEvent(int repositoryType, String repositoryName,
- String user, Date eventTime, String accessType,
- String resourcePath, String resourceType, String action,
- short accessResult, String agentId, long policyId,
- String resultReason, String aclEnforcer, String sessionId,
- String clientType, String clientIP, String requestData, String clusterName, String zoneName) {
- this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
- policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, zoneName, null);
-
- }
-
- public AuthzAuditEvent(int repositoryType, String repositoryName,
- String user, Date eventTime, String accessType,
- String resourcePath, String resourceType, String action,
- short accessResult, String agentId, long policyId,
- String resultReason, String aclEnforcer, String sessionId,
- String clientType, String clientIP, String requestData, String clusterName, String zoneName, Long policyVersion) {
- this.repositoryType = repositoryType;
- this.repositoryName = repositoryName;
- this.user = user;
- this.eventTime = eventTime;
- this.accessType = accessType;
- this.resourcePath = resourcePath;
- this.resourceType = resourceType;
- this.action = action;
- this.accessResult = accessResult;
- this.agentId = agentId;
- this.policyId = policyId;
- this.resultReason = resultReason;
- this.aclEnforcer = aclEnforcer;
- this.sessionId = sessionId;
- this.clientType = clientType;
- this.clientIP = clientIP;
- this.requestData = requestData;
- this.clusterName = clusterName;
- this.zoneName = zoneName;
- this.policyVersion = policyVersion;
- }
-
- /**
- * @return the repositoryType
- */
- public int getRepositoryType() {
- return repositoryType;
- }
-
- /**
- * @param repositoryType
- * the repositoryType to set
- */
- public void setRepositoryType(int repositoryType) {
- this.repositoryType = repositoryType;
- }
-
- /**
- * @return the repositoryName
- */
- public String getRepositoryName() {
- return repositoryName;
- }
-
- /**
- * @param repositoryName
- * the repositoryName to set
- */
- public void setRepositoryName(String repositoryName) {
- this.repositoryName = repositoryName;
- }
-
- /**
- * @return the user
- */
- public String getUser() {
- return user;
- }
-
- /**
- * @param user
- * the user to set
- */
- public void setUser(String user) {
- this.user = user;
- }
-
- /**
- * @return the timeStamp
- */
- public Date getEventTime() {
- return eventTime;
- }
-
- /**
- * @param eventTime
- * the eventTime to set
- */
- public void setEventTime(Date eventTime) {
- this.eventTime = eventTime;
- }
-
- /**
- * @return the accessType
- */
- public String getAccessType() {
- return accessType;
- }
-
- /**
- * @param accessType
- * the accessType to set
- */
- public void setAccessType(String accessType) {
- this.accessType = accessType;
- }
-
- /**
- * @return the resourcePath
- */
- public String getResourcePath() {
- return resourcePath;
- }
-
- /**
- * @param resourcePath
- * the resourcePath to set
- */
- public void setResourcePath(String resourcePath) {
- this.resourcePath = resourcePath;
- }
-
- /**
- * @return the resourceType
- */
- public String getResourceType() {
- return resourceType;
- }
-
- /**
- * @param resourceType
- * the resourceType to set
- */
- public void setResourceType(String resourceType) {
- this.resourceType = resourceType;
- }
-
- /**
- * @return the action
- */
- public String getAction() { return action; }
-
- /**
- * @param action
- * the action to set
- */
- public void setAction(String action) {
- this.action = action;
- }
-
- /**
- * @return the accessResult
- */
- public short getAccessResult() {
- return accessResult;
- }
-
- /**
- * @param accessResult
- * the accessResult to set
- */
- public void setAccessResult(short accessResult) {
- this.accessResult = accessResult;
- }
-
- /**
- * @return the agentId
- */
- public String getAgentId() {
- return agentId;
- }
-
- /**
- * @param agentId
- * the agentId to set
- */
- public void setAgentId(String agentId) {
- this.agentId = agentId;
- }
-
- /**
- * @return the policyId
- */
- public long getPolicyId() {
- return policyId;
- }
-
- /**
- * @param policyId
- * the policyId to set
- */
- public void setPolicyId(long policyId) {
- this.policyId = policyId;
- }
-
- /**
- * @return the resultReason
- */
- public String getResultReason() {
- return resultReason;
- }
-
- /**
- * @param resultReason
- * the resultReason to set
- */
- public void setResultReason(String resultReason) {
- this.resultReason = resultReason;
- }
-
- /**
- * @return the aclEnforcer
- */
- public String getAclEnforcer() {
- return aclEnforcer;
- }
-
- /**
- * @param aclEnforcer
- * the aclEnforcer to set
- */
- public void setAclEnforcer(String aclEnforcer) {
- this.aclEnforcer = aclEnforcer;
- }
-
- /**
- * @return the sessionId
- */
- public String getSessionId() {
- return sessionId;
- }
-
- /**
- * @param sessionId
- * the sessionId to set
- */
- public void setSessionId(String sessionId) {
- this.sessionId = sessionId;
- }
-
- /**
- * @return the clientType
- */
- public String getClientType() {
- return clientType;
- }
-
- /**
- * @param clientType
- * the clientType to set
- */
- public void setClientType(String clientType) {
- this.clientType = clientType;
- }
-
- /**
- * @return the clientIP
- */
- public String getClientIP() {
- return clientIP;
- }
-
- /**
- * @param clientIP
- * the clientIP to set
- */
- public void setClientIP(String clientIP) {
- this.clientIP = clientIP;
- }
-
- /**
- * @return the requestData
- */
- public String getRequestData() { return requestData; }
-
- /**
- * @param requestData
- * the requestData to set
- */
- public void setRequestData(String requestData) {
- this.requestData = requestData;
- }
-
- public String getAgentHostname() {
- return agentHostname;
- }
-
- public void setAgentHostname(String agentHostname) {
- this.agentHostname = agentHostname;
- }
-
- public String getLogType() {
- return logType;
- }
-
- public void setLogType(String logType) {
- this.logType = logType;
- }
-
- public String getEventId() {
- return eventId;
- }
-
- public void setEventId(String eventId) {
- this.eventId = eventId;
- }
-
- public long getSeqNum() {
- return seqNum;
- }
-
- public void setSeqNum(long seqNum) {
- this.seqNum = seqNum;
- }
-
- public long getEventCount() {
- return eventCount;
- }
-
- public void setEventCount(long frequencyCount) {
- this.eventCount = frequencyCount;
- }
-
- public long getEventDurationMS() {
- return eventDurationMS;
- }
-
- public void setEventDurationMS(long frequencyDurationMS) {
- this.eventDurationMS = frequencyDurationMS;
- }
-
- public Set<String> getTags() {
- return tags;
- }
-
- public void setTags(Set<String> tags) {
- this.tags = tags;
- }
-
- public Set<String> getDatasets() {
- return datasets;
- }
-
- public void setDatasets(Set<String> datasets) {
- this.datasets = datasets;
- }
-
- public Set<String> getProjects() {
- return projects;
- }
-
- public void setProjects(Set<String> projects) {
- this.projects = projects;
- }
-
- public String getClusterName() {
- return clusterName;
- }
-
- public void setZoneName(String zoneName) {
- this.zoneName = zoneName;
- }
-
- public String getZoneName() {
- return zoneName;
- }
-
- public void setPolicyVersion(Long policyVersion) {
- this.policyVersion = policyVersion;
- }
-
- public Long getPolicyVersion() {
- return policyVersion;
- }
-
- public void setClusterName(String clusterName) {
- this.clusterName = clusterName;
- }
-
- public String getAdditionalInfo() { return this.additionalInfo; }
-
- public void setAdditionalInfo(String additionalInfo) { this.additionalInfo = additionalInfo; }
-
- @JsonIgnore
- @Override
- public String getEventKey() {
- String key = user + "^" + accessType + "^" + resourcePath + "^"
- + resourceType + "^" + action + "^" + accessResult + "^"
- + sessionId + "^" + clientIP;
- return key;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
-
- sb.append("AuthzAuditEvent{");
- toString(sb);
- sb.append("}");
-
- return sb.toString();
- }
-
- protected StringBuilder toString(StringBuilder sb) {
- sb.append("repositoryType=").append(repositoryType)
- .append(FIELD_SEPARATOR).append("repositoryName=")
- .append(repositoryName).append(FIELD_SEPARATOR).append("user=")
- .append(user).append(FIELD_SEPARATOR).append("eventTime=")
- .append(eventTime).append(FIELD_SEPARATOR)
- .append("accessType=").append(accessType)
- .append(FIELD_SEPARATOR).append("resourcePath=")
- .append(resourcePath).append(FIELD_SEPARATOR)
- .append("resourceType=").append(resourceType)
- .append(FIELD_SEPARATOR).append("action=").append(action)
- .append(FIELD_SEPARATOR).append("accessResult=")
- .append(accessResult).append(FIELD_SEPARATOR)
- .append("agentId=").append(agentId).append(FIELD_SEPARATOR)
- .append("policyId=").append(policyId).append(FIELD_SEPARATOR)
- .append("resultReason=").append(resultReason)
- .append(FIELD_SEPARATOR).append("aclEnforcer=")
- .append(aclEnforcer).append(FIELD_SEPARATOR)
- .append("sessionId=").append(sessionId).append(FIELD_SEPARATOR)
- .append("clientType=").append(clientType)
- .append(FIELD_SEPARATOR).append("clientIP=").append(clientIP)
- .append(FIELD_SEPARATOR).append("requestData=")
- .append(requestData).append(FIELD_SEPARATOR)
- .append("agentHostname=").append(agentHostname)
- .append(FIELD_SEPARATOR).append("logType=").append(logType)
- .append(FIELD_SEPARATOR).append("eventId=").append(eventId)
- .append(FIELD_SEPARATOR).append("seq_num=").append(seqNum)
- .append(FIELD_SEPARATOR).append("event_count=")
- .append(eventCount).append(FIELD_SEPARATOR)
- .append("event_dur_ms=").append(eventDurationMS)
- .append(FIELD_SEPARATOR).append("tags=").append("[").append(StringUtils.join(tags, ", ")).append("]")
- .append(FIELD_SEPARATOR).append("datasets=").append("[").append(datasets != null ? StringUtils.join(datasets, ", ") : "").append("]")
- .append(FIELD_SEPARATOR).append("projects=").append("[").append(projects != null ? StringUtils.join(projects, ", ") : "").append("]")
- .append(FIELD_SEPARATOR).append("clusterName=").append(clusterName)
- .append(FIELD_SEPARATOR).append("zoneName=").append(zoneName)
- .append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion)
- .append(FIELD_SEPARATOR).append("additionalInfo=").append(additionalInfo);
-
- return sb;
- }
+ @JsonProperty("sess")
+ protected String sessionId;
+
+ @JsonProperty("cliType")
+ protected String clientType;
+
+ @JsonProperty("cliIP")
+ protected String clientIP;
+
+ @JsonProperty("reqData")
+ protected String requestData;
+
+ @JsonProperty("agentHost")
+ protected String agentHostname;
+
+ @JsonProperty("logType")
+ protected String logType;
+
+ @JsonProperty("id")
+ protected String eventId;
+
+ /**
+ * This to ensure order within a session. Order not guaranteed across
+ * processes and hosts
+ */
+ @JsonProperty("seq_num")
+ protected long seqNum;
+
+ @JsonProperty("event_count")
+ protected long eventCount = 1;
+
+ @JsonProperty("event_dur_ms")
+ protected long eventDurationMS;
+
+ @JsonProperty("tags")
+ protected Set<String> tags = new HashSet<>();
+
+ @JsonProperty("datasets")
+ protected Set<String> datasets;
+
+ @JsonProperty("projects")
+ protected Set<String> projects;
+
+ @JsonProperty("additional_info")
+ protected String additionalInfo;
+
+ @JsonProperty("cluster_name")
+ protected String clusterName;
+
+ @JsonProperty("zone_name")
+ protected String zoneName;
+
+ @JsonProperty("policy_version")
+ protected Long policyVersion;
+
+ public AuthzAuditEvent() {
+ super();
+
+ this.repositoryType = 0;
+ }
+
+ public AuthzAuditEvent(int repositoryType, String repositoryName,
+ String user, Date eventTime, String accessType,
+ String resourcePath, String resourceType, String action,
+ short accessResult, String agentId, long policyId,
+ String resultReason, String aclEnforcer, String sessionId,
+ String clientType, String clientIP, String requestData, String clusterName) {
+ this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
+ policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, null);
+ }
+
+ public AuthzAuditEvent(int repositoryType, String repositoryName,
+ String user, Date eventTime, String accessType,
+ String resourcePath, String resourceType, String action,
+ short accessResult, String agentId, long policyId,
+ String resultReason, String aclEnforcer, String sessionId,
+ String clientType, String clientIP, String requestData, String clusterName, String zoneName) {
+ this(repositoryType, repositoryName, user, eventTime, accessType, resourcePath, resourceType, action, accessResult, agentId,
+ policyId, resultReason, aclEnforcer, sessionId, clientType, clientIP, requestData, clusterName, zoneName, null);
+ }
+
+ public AuthzAuditEvent(int repositoryType, String repositoryName,
+ String user, Date eventTime, String accessType,
+ String resourcePath, String resourceType, String action,
+ short accessResult, String agentId, long policyId,
+ String resultReason, String aclEnforcer, String sessionId,
+ String clientType, String clientIP, String requestData, String clusterName, String zoneName, Long policyVersion) {
+ this.repositoryType = repositoryType;
+ this.repositoryName = repositoryName;
+ this.user = user;
+ this.eventTime = eventTime;
+ this.accessType = accessType;
+ this.resourcePath = resourcePath;
+ this.resourceType = resourceType;
+ this.action = action;
+ this.accessResult = accessResult;
+ this.agentId = agentId;
+ this.policyId = policyId;
+ this.resultReason = resultReason;
+ this.aclEnforcer = aclEnforcer;
+ this.sessionId = sessionId;
+ this.clientType = clientType;
+ this.clientIP = clientIP;
+ this.requestData = requestData;
+ this.clusterName = clusterName;
+ this.zoneName = zoneName;
+ this.policyVersion = policyVersion;
+ }
+
+ /**
+ * @return the repositoryType
+ */
+ public int getRepositoryType() {
+ return repositoryType;
+ }
+
+ /**
+ * @param repositoryType the repositoryType to set
+ */
+ public void setRepositoryType(int repositoryType) {
+ this.repositoryType = repositoryType;
+ }
+
+ /**
+ * @return the repositoryName
+ */
+ public String getRepositoryName() {
+ return repositoryName;
+ }
+
+ /**
+ * @param repositoryName the repositoryName to set
+ */
+ public void setRepositoryName(String repositoryName) {
+ this.repositoryName = repositoryName;
+ }
+
+ /**
+ * @return the user
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * @param user the user to set
+ */
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ /**
+ * @return the accessType
+ */
+ public String getAccessType() {
+ return accessType;
+ }
+
+ /**
+ * @param accessType the accessType to set
+ */
+ public void setAccessType(String accessType) {
+ this.accessType = accessType;
+ }
+
+ /**
+ * @return the resourcePath
+ */
+ public String getResourcePath() {
+ return resourcePath;
+ }
+
+ /**
+ * @param resourcePath the resourcePath to set
+ */
+ public void setResourcePath(String resourcePath) {
+ this.resourcePath = resourcePath;
+ }
+
+ /**
+ * @return the resourceType
+ */
+ public String getResourceType() {
+ return resourceType;
+ }
+
+ /**
+ * @param resourceType the resourceType to set
+ */
+ public void setResourceType(String resourceType) {
+ this.resourceType = resourceType;
+ }
+
+ /**
+ * @return the action
+ */
+ public String getAction() {
+ return action;
+ }
+
+ /**
+ * @param action the action to set
+ */
+ public void setAction(String action) {
+ this.action = action;
+ }
+
+ /**
+ * @return the accessResult
+ */
+ public short getAccessResult() {
+ return accessResult;
+ }
+
+ /**
+ * @param accessResult the accessResult to set
+ */
+ public void setAccessResult(short accessResult) {
+ this.accessResult = accessResult;
+ }
+
+ /**
+ * @return the agentId
+ */
+ public String getAgentId() {
+ return agentId;
+ }
+
+ /**
+ * @param agentId the agentId to set
+ */
+ public void setAgentId(String agentId) {
+ this.agentId = agentId;
+ }
+
+ /**
+ * @return the policyId
+ */
+ public long getPolicyId() {
+ return policyId;
+ }
+
+ /**
+ * @param policyId the policyId to set
+ */
+ public void setPolicyId(long policyId) {
+ this.policyId = policyId;
+ }
+
+ /**
+ * @return the resultReason
+ */
+ public String getResultReason() {
+ return resultReason;
+ }
+
+ /**
+ * @param resultReason the resultReason to set
+ */
+ public void setResultReason(String resultReason) {
+ this.resultReason = resultReason;
+ }
+
+ /**
+ * @return the aclEnforcer
+ */
+ public String getAclEnforcer() {
+ return aclEnforcer;
+ }
+
+ /**
+ * @param aclEnforcer the aclEnforcer to set
+ */
+ public void setAclEnforcer(String aclEnforcer) {
+ this.aclEnforcer = aclEnforcer;
+ }
+
+ /**
+ * @return the sessionId
+ */
+ public String getSessionId() {
+ return sessionId;
+ }
+
+ /**
+ * @param sessionId the sessionId to set
+ */
+ public void setSessionId(String sessionId) {
+ this.sessionId = sessionId;
+ }
+
+ /**
+ * @return the clientType
+ */
+ public String getClientType() {
+ return clientType;
+ }
+
+ /**
+ * @param clientType the clientType to set
+ */
+ public void setClientType(String clientType) {
+ this.clientType = clientType;
+ }
+
+ /**
+ * @return the clientIP
+ */
+ public String getClientIP() {
+ return clientIP;
+ }
+
+ /**
+ * @param clientIP the clientIP to set
+ */
+ public void setClientIP(String clientIP) {
+ this.clientIP = clientIP;
+ }
+
+ /**
+ * @return the requestData
+ */
+ public String getRequestData() {
+ return requestData;
+ }
+
+ /**
+ * @param requestData the requestData to set
+ */
+ public void setRequestData(String requestData) {
+ this.requestData = requestData;
+ }
+
+ public String getAgentHostname() {
+ return agentHostname;
+ }
+
+ public void setAgentHostname(String agentHostname) {
+ this.agentHostname = agentHostname;
+ }
+
+ public String getLogType() {
+ return logType;
+ }
+
+ public void setLogType(String logType) {
+ this.logType = logType;
+ }
+
+ public String getEventId() {
+ return eventId;
+ }
+
+ public void setEventId(String eventId) {
+ this.eventId = eventId;
+ }
+
+ public long getSeqNum() {
+ return seqNum;
+ }
+
+ public void setSeqNum(long seqNum) {
+ this.seqNum = seqNum;
+ }
+
+ public long getEventCount() {
+ return eventCount;
+ }
+
+ public void setEventCount(long frequencyCount) {
+ this.eventCount = frequencyCount;
+ }
+
+ public long getEventDurationMS() {
+ return eventDurationMS;
+ }
+
+ public void setEventDurationMS(long frequencyDurationMS) {
+ this.eventDurationMS = frequencyDurationMS;
+ }
+
+ public Set<String> getTags() {
+ return tags;
+ }
+
+ public void setTags(Set<String> tags) {
+ this.tags = tags;
+ }
+
+ public Set<String> getDatasets() {
+ return datasets;
+ }
+
+ public void setDatasets(Set<String> datasets) {
+ this.datasets = datasets;
+ }
+
+ public Set<String> getProjects() {
+ return projects;
+ }
+
+ public void setProjects(Set<String> projects) {
+ this.projects = projects;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public String getZoneName() {
+ return zoneName;
+ }
+
+ public void setZoneName(String zoneName) {
+ this.zoneName = zoneName;
+ }
+
+ public Long getPolicyVersion() {
+ return policyVersion;
+ }
+
+ public void setPolicyVersion(Long policyVersion) {
+ this.policyVersion = policyVersion;
+ }
+
+ public String getAdditionalInfo() {
+ return this.additionalInfo;
+ }
+
+ public void setAdditionalInfo(String additionalInfo) {
+ this.additionalInfo = additionalInfo;
+ }
+
+ @JsonIgnore
+ @Override
+ public String getEventKey() {
+ return user + "^" + accessType + "^" + resourcePath + "^" + resourceType + "^" + action + "^" + accessResult + "^" + sessionId + "^" + clientIP;
+ }
+
+ /**
+ * @return the timeStamp
+ */
+ public Date getEventTime() {
+ return eventTime;
+ }
+
+ /**
+ * @param eventTime the eventTime to set
+ */
+ public void setEventTime(Date eventTime) {
+ this.eventTime = eventTime;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("AuthzAuditEvent{");
+ toString(sb);
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+ protected StringBuilder toString(StringBuilder sb) {
+ sb.append("repositoryType=").append(repositoryType)
+ .append(FIELD_SEPARATOR).append("repositoryName=")
+ .append(repositoryName).append(FIELD_SEPARATOR).append("user=")
+ .append(user).append(FIELD_SEPARATOR).append("eventTime=")
+ .append(eventTime).append(FIELD_SEPARATOR)
+ .append("accessType=").append(accessType)
+ .append(FIELD_SEPARATOR).append("resourcePath=")
+ .append(resourcePath).append(FIELD_SEPARATOR)
+ .append("resourceType=").append(resourceType)
+ .append(FIELD_SEPARATOR).append("action=").append(action)
+ .append(FIELD_SEPARATOR).append("accessResult=")
+ .append(accessResult).append(FIELD_SEPARATOR)
+ .append("agentId=").append(agentId).append(FIELD_SEPARATOR)
+ .append("policyId=").append(policyId).append(FIELD_SEPARATOR)
+ .append("resultReason=").append(resultReason)
+ .append(FIELD_SEPARATOR).append("aclEnforcer=")
+ .append(aclEnforcer).append(FIELD_SEPARATOR)
+ .append("sessionId=").append(sessionId).append(FIELD_SEPARATOR)
+ .append("clientType=").append(clientType)
+ .append(FIELD_SEPARATOR).append("clientIP=").append(clientIP)
+ .append(FIELD_SEPARATOR).append("requestData=")
+ .append(requestData).append(FIELD_SEPARATOR)
+ .append("agentHostname=").append(agentHostname)
+ .append(FIELD_SEPARATOR).append("logType=").append(logType)
+ .append(FIELD_SEPARATOR).append("eventId=").append(eventId)
+ .append(FIELD_SEPARATOR).append("seq_num=").append(seqNum)
+ .append(FIELD_SEPARATOR).append("event_count=")
+ .append(eventCount).append(FIELD_SEPARATOR)
+ .append("event_dur_ms=").append(eventDurationMS)
+ .append(FIELD_SEPARATOR).append("tags=").append("[").append(StringUtils.join(tags, ", ")).append("]")
+ .append(FIELD_SEPARATOR).append("datasets=").append("[").append(datasets != null ? StringUtils.join(datasets, ", ") : "").append("]")
+ .append(FIELD_SEPARATOR).append("projects=").append("[").append(projects != null ? StringUtils.join(projects, ", ") : "").append("]")
+ .append(FIELD_SEPARATOR).append("clusterName=").append(clusterName)
+ .append(FIELD_SEPARATOR).append("zoneName=").append(zoneName)
+ .append(FIELD_SEPARATOR).append("policyVersion=").append(policyVersion)
+ .append(FIELD_SEPARATOR).append("additionalInfo=").append(additionalInfo);
+
+ return sb;
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/EnumRepositoryType.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/EnumRepositoryType.java
index a8364c201..28b30f868 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/EnumRepositoryType.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/EnumRepositoryType.java
@@ -17,21 +17,22 @@
* under the License.
*/
- package org.apache.ranger.audit.model;
+package org.apache.ranger.audit.model;
public final class EnumRepositoryType {
-
- public static final int HDFS = 1;
-
- public static final int HBASE = 2;
-
- public static final int HIVE = 3;
-
- public static final int XAAGENT = 4;
-
- public static final int KNOX = 5;
-
- public static final int STORM = 6;
-
-
+ public static final int HDFS = 1;
+
+ public static final int HBASE = 2;
+
+ public static final int HIVE = 3;
+
+ public static final int XAAGENT = 4;
+
+ public static final int KNOX = 5;
+
+ public static final int STORM = 6;
+
+ private EnumRepositoryType() {
+ // to block instantiation
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/SPOOL_FILE_STATUS.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/SPOOL_FILE_STATUS.java
index 3765c9c92..5156cf27b 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/SPOOL_FILE_STATUS.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/SPOOL_FILE_STATUS.java
@@ -20,5 +20,5 @@
package org.apache.ranger.audit.model;
public enum SPOOL_FILE_STATUS {
- pending, write_inprogress, read_inprogress, done
+ pending, write_inprogress, read_inprogress, done
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AsyncAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AsyncAuditProvider.java
index 2a3ec77b6..ea3d4f7ad 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AsyncAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AsyncAuditProvider.java
@@ -17,273 +17,273 @@
* under the License.
*/
- package org.apache.ranger.audit.provider;
+package org.apache.ranger.audit.provider;
-import java.util.concurrent.BlockingQueue;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.Properties;
-import org.apache.ranger.audit.model.AuditEventBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+public class AsyncAuditProvider extends MultiDestAuditProvider implements Runnable {
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncAuditProvider.class);
+
+ private static final int mStopLoopIntervalSecs = 1; // 1 second
+ private static final int mWaitToCompleteLoopIntervalSecs = 1; // 1 second
+ private static int sThreadCount;
+
+ private final BlockingQueue<AuditEventBase> mQueue;
+ private final String mName;
+
+ // Summary of logs handled
+ private final AtomicLong lifeTimeInLogCount = new AtomicLong(0); // Total count, including drop count
+ private final AtomicLong lifeTimeOutLogCount = new AtomicLong(0);
+ private final AtomicLong lifeTimeDropCount = new AtomicLong(0);
+ private final AtomicLong intervalInLogCount = new AtomicLong(0);
+ private final AtomicLong intervalOutLogCount = new AtomicLong(0);
+ private final AtomicLong intervalDropCount = new AtomicLong(0);
+
+ private Thread mThread;
+ private int mMaxQueueSize = 10 * 1024;
+ private int mMaxFlushInterval = 5000; // 5 seconds
+ private long lastIntervalLogTime = System.currentTimeMillis();
+ private int intervalLogDurationMS = 60000;
+ private long lastFlushTime = System.currentTimeMillis();
+
+ public AsyncAuditProvider(String name, int maxQueueSize, int maxFlushInterval) {
+ LOG.info("AsyncAuditProvider({}): creating..", name);
+
+ if (maxQueueSize < 1) {
+ LOG.warn("AsyncAuditProvider({}): invalid maxQueueSize={}. will use default {}", name, maxQueueSize, mMaxQueueSize);
+
+ maxQueueSize = mMaxQueueSize;
+ }
+
+ mName = name;
+ mMaxQueueSize = maxQueueSize;
+ mMaxFlushInterval = maxFlushInterval;
+
+ mQueue = new ArrayBlockingQueue<>(mMaxQueueSize);
+ }
+
+ public AsyncAuditProvider(String name, int maxQueueSize, int maxFlushInterval, AuditHandler provider) {
+ this(name, maxQueueSize, maxFlushInterval);
+
+ addAuditProvider(provider);
+ }
+
+ @Override
+ public void init(Properties props) {
+ LOG.info("AsyncAuditProvider({}).init()", mName);
+
+ super.init(props);
+ }
+
+ @Override
+ public boolean log(AuditEventBase event) {
+ LOG.debug("AsyncAuditProvider.logEvent(AuditEventBase)");
+
+ queueEvent(event);
+ return true;
+ }
+
+ @Override
+ public void start() {
+ mThread = new Thread(this, "AsyncAuditProvider" + (++sThreadCount));
+
+ mThread.setDaemon(true);
+ mThread.start();
+
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("==> AsyncAuditProvider.stop()");
+
+ try {
+ LOG.info("Interrupting child thread of {}...", mName);
+
+ mThread.interrupt();
+
+ while (mThread.isAlive()) {
+ try {
+ LOG.info("Waiting for child thread of {} to exit. Sleeping for {} secs", mName, mStopLoopIntervalSecs);
+
+ mThread.join(mStopLoopIntervalSecs * 1000L);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for child thread to join! Proceeding with stop", e);
+ break;
+ }
+ }
+
+ super.stop();
+ } finally {
+ LOG.info("<== AsyncAuditProvider.stop()");
+ }
+ }
+
+ @Override
+ public void waitToComplete() {
+ waitToComplete(0);
+
+ super.waitToComplete();
+ }
+
+ public void waitToComplete(long maxWaitSeconds) {
+ LOG.debug("==> AsyncAuditProvider.waitToComplete()");
+
+ try {
+ for (long waitTime = 0; !isEmpty() && (maxWaitSeconds <= 0 || maxWaitSeconds > waitTime); waitTime += mWaitToCompleteLoopIntervalSecs) {
+ try {
+ LOG.info("{} messages yet to be flushed by {}. Sleeoping for {} sec", mQueue.size(), mName, mWaitToCompleteLoopIntervalSecs);
+
+ Thread.sleep(mWaitToCompleteLoopIntervalSecs * 1000L);
+ } catch (InterruptedException excp) {
+ // someone really wants service to exit, abandon unwritten audits and exit.
+ LOG.warn("Caught interrupted exception! {} messages still unflushed! Won't wait for queue to flush, exiting...", mQueue.size(), excp);
+
+ break;
+ }
+ }
+ } finally {
+ LOG.debug("<== AsyncAuditProvider.waitToComplete()");
+ }
+ }
+
+ public int getIntervalLogDurationMS() {
+ return intervalLogDurationMS;
+ }
+
+ public void setIntervalLogDurationMS(int intervalLogDurationMS) {
+ this.intervalLogDurationMS = intervalLogDurationMS;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("==> AsyncAuditProvider.run()");
+
+ while (true) {
+ AuditEventBase event = null;
+ try {
+ event = dequeueEvent();
+
+ if (event != null) {
+ super.log(event);
+ } else {
+ lastFlushTime = System.currentTimeMillis();
+
+ flush();
+ }
+ } catch (InterruptedException excp) {
+ LOG.info("AsyncAuditProvider.run - Interrupted! Breaking out of while loop.");
+
+ break;
+ } catch (Exception excp) {
+ logFailedEvent(event, excp);
+ }
+ }
+
+ try {
+ lastFlushTime = System.currentTimeMillis();
+
+ flush();
+ } catch (Exception excp) {
+ LOG.error("AsyncAuditProvider.run()", excp);
+ }
+
+ LOG.info("<== AsyncAuditProvider.run()");
+ }
+
+ private void queueEvent(AuditEventBase event) {
+ // Increase counts
+ lifeTimeInLogCount.incrementAndGet();
+ intervalInLogCount.incrementAndGet();
+
+ if (!mQueue.offer(event)) {
+ lifeTimeDropCount.incrementAndGet();
+ intervalDropCount.incrementAndGet();
+ }
+ }
+
+ private AuditEventBase dequeueEvent() throws InterruptedException {
+ AuditEventBase ret = mQueue.poll();
+
+ while (ret == null) {
+ logSummaryIfRequired();
+
+ if (mMaxFlushInterval > 0) {
+ long timeTillNextFlush = getTimeTillNextFlush();
+
+ if (timeTillNextFlush <= 0) {
+ break; // force flush
+ }
+
+ ret = mQueue.poll(timeTillNextFlush, TimeUnit.MILLISECONDS);
+ } else {
+ // Let's wake up for summary logging
+ long waitTime = intervalLogDurationMS - (System.currentTimeMillis() - lastIntervalLogTime);
+
+ if (waitTime <= 0) {
+ waitTime = intervalLogDurationMS;
+ }
+
+ ret = mQueue.poll(waitTime, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ if (ret != null) {
+ lifeTimeOutLogCount.incrementAndGet();
+ intervalOutLogCount.incrementAndGet();
+ }
+
+ logSummaryIfRequired();
+
+ return ret;
+ }
+
+ private void logSummaryIfRequired() {
+ long intervalSinceLastLog = System.currentTimeMillis() - lastIntervalLogTime;
+
+ if (intervalSinceLastLog > intervalLogDurationMS) {
+ if (intervalInLogCount.get() > 0 || intervalOutLogCount.get() > 0) {
+ long queueSize = mQueue.size();
+
+ LOG.info("AsyncAuditProvider-stats:{}: past {}: inLogs={}, outLogs={}, dropped={}, currentQueueSize={}", mName, formatIntervalForLog(intervalSinceLastLog), intervalInLogCount, intervalOutLogCount, intervalDropCount, queueSize);
+ LOG.info("AsyncAuditProvider-stats:{}: process lifetime: inLogs={}, outLogs={}, dropped={}", mName, lifeTimeInLogCount, lifeTimeOutLogCount, lifeTimeDropCount);
+ }
+
+ lastIntervalLogTime = System.currentTimeMillis();
+
+ intervalInLogCount.set(0);
+ intervalOutLogCount.set(0);
+ intervalDropCount.set(0);
+ }
+ }
+
+ private boolean isEmpty() {
+ return mQueue.isEmpty();
+ }
+
+ private long getTimeTillNextFlush() {
+ long timeTillNextFlush = mMaxFlushInterval;
+
+ if (mMaxFlushInterval > 0) {
+ if (lastFlushTime != 0) {
+ long timeSinceLastFlush = System.currentTimeMillis() - lastFlushTime;
+
+ if (timeSinceLastFlush >= mMaxFlushInterval) {
+ timeTillNextFlush = 0;
+ } else {
+ timeTillNextFlush = mMaxFlushInterval - timeSinceLastFlush;
+ }
+ }
+ }
-public class AsyncAuditProvider extends MultiDestAuditProvider implements
- Runnable {
-
- private static final Logger LOG = LoggerFactory.getLogger(AsyncAuditProvider.class);
-
- private static int sThreadCount = 0;
-
- private BlockingQueue<AuditEventBase> mQueue = null;
- private Thread mThread = null;
- private String mName = null;
- private int mMaxQueueSize = 10 * 1024;
- private int mMaxFlushInterval = 5000; // 5 seconds
-
- private static final int mStopLoopIntervalSecs = 1; // 1 second
- private static final int mWaitToCompleteLoopIntervalSecs = 1; // 1 second
-
- // Summary of logs handled
- private AtomicLong lifeTimeInLogCount = new AtomicLong(0); // Total count, including drop count
- private AtomicLong lifeTimeOutLogCount = new AtomicLong(0);
- private AtomicLong lifeTimeDropCount = new AtomicLong(0);
- private AtomicLong intervalInLogCount = new AtomicLong(0);
- private AtomicLong intervalOutLogCount = new AtomicLong(0);
- private AtomicLong intervalDropCount = new AtomicLong(0);
- private long lastIntervalLogTime = System.currentTimeMillis();
- private int intervalLogDurationMS = 60000;
- private long lastFlushTime = System.currentTimeMillis();
-
- public AsyncAuditProvider(String name, int maxQueueSize, int maxFlushInterval) {
- LOG.info("AsyncAuditProvider(" + name + "): creating..");
-
- if(maxQueueSize < 1) {
- LOG.warn("AsyncAuditProvider(" + name + "): invalid maxQueueSize=" + maxQueueSize + ". will use default " + mMaxQueueSize);
-
- maxQueueSize = mMaxQueueSize;
- }
-
- mName = name;
- mMaxQueueSize = maxQueueSize;
- mMaxFlushInterval = maxFlushInterval;
-
- mQueue = new ArrayBlockingQueue<AuditEventBase>(mMaxQueueSize);
- }
-
- public AsyncAuditProvider(String name, int maxQueueSize, int maxFlushInterval, AuditHandler provider) {
- this(name, maxQueueSize, maxFlushInterval);
-
- addAuditProvider(provider);
- }
-
- @Override
- public void init(Properties props) {
- LOG.info("AsyncAuditProvider(" + mName + ").init()");
-
- super.init(props);
- }
-
- public int getIntervalLogDurationMS() {
- return intervalLogDurationMS;
- }
-
- public void setIntervalLogDurationMS(int intervalLogDurationMS) {
- this.intervalLogDurationMS = intervalLogDurationMS;
- }
-
- @Override
- public boolean log(AuditEventBase event) {
- LOG.debug("AsyncAuditProvider.logEvent(AuditEventBase)");
-
- queueEvent(event);
- return true;
- }
-
- @Override
- public void start() {
- mThread = new Thread(this, "AsyncAuditProvider" + (++sThreadCount));
-
- mThread.setDaemon(true);
- mThread.start();
-
- super.start();
- }
-
- @Override
- public void stop() {
- LOG.info("==> AsyncAuditProvider.stop()");
- try {
- LOG.info("Interrupting child thread of " + mName + "..." );
- mThread.interrupt();
- while (mThread.isAlive()) {
- try {
- LOG.info(String.format("Waiting for child thread of %s to exit. Sleeping for %d secs", mName, mStopLoopIntervalSecs));
- mThread.join(mStopLoopIntervalSecs * 1000L);
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting for child thread to join! Proceeding with stop", e);
- break;
- }
- }
-
- super.stop();
- } finally {
- LOG.info("<== AsyncAuditProvider.stop()");
- }
- }
-
- @Override
- public void waitToComplete() {
- waitToComplete(0);
-
- super.waitToComplete();
- }
-
- @Override
- public void run() {
- LOG.info("==> AsyncAuditProvider.run()");
-
- while (true) {
- AuditEventBase event = null;
- try {
- event = dequeueEvent();
-
- if (event != null) {
- super.log(event);
- } else {
- lastFlushTime = System.currentTimeMillis();
- flush();
- }
- } catch (InterruptedException excp) {
- LOG.info("AsyncAuditProvider.run - Interrupted! Breaking out of while loop.");
- break;
- } catch (Exception excp) {
- logFailedEvent(event, excp);
- }
- }
-
- try {
- lastFlushTime = System.currentTimeMillis();
- flush();
- } catch (Exception excp) {
- LOG.error("AsyncAuditProvider.run()", excp);
- }
-
- LOG.info("<== AsyncAuditProvider.run()");
- }
-
- private void queueEvent(AuditEventBase event) {
- // Increase counts
- lifeTimeInLogCount.incrementAndGet();
- intervalInLogCount.incrementAndGet();
-
- if(! mQueue.offer(event)) {
- lifeTimeDropCount.incrementAndGet();
- intervalDropCount.incrementAndGet();
- }
- }
-
- private AuditEventBase dequeueEvent() throws InterruptedException {
- AuditEventBase ret = mQueue.poll();
-
- while(ret == null) {
- logSummaryIfRequired();
-
- if (mMaxFlushInterval > 0 ) {
- long timeTillNextFlush = getTimeTillNextFlush();
-
- if (timeTillNextFlush <= 0) {
- break; // force flush
- }
-
- ret = mQueue.poll(timeTillNextFlush, TimeUnit.MILLISECONDS);
- } else {
- // Let's wake up for summary logging
- long waitTime = intervalLogDurationMS - (System.currentTimeMillis() - lastIntervalLogTime);
- waitTime = waitTime <= 0 ? intervalLogDurationMS : waitTime;
-
- ret = mQueue.poll(waitTime, TimeUnit.MILLISECONDS);
- }
- }
-
- if(ret != null) {
- lifeTimeOutLogCount.incrementAndGet();
- intervalOutLogCount.incrementAndGet();
- }
-
- logSummaryIfRequired();
-
- return ret;
- }
-
- private void logSummaryIfRequired() {
- long intervalSinceLastLog = System.currentTimeMillis() - lastIntervalLogTime;
-
- if (intervalSinceLastLog > intervalLogDurationMS) {
- if (intervalInLogCount.get() > 0 || intervalOutLogCount.get() > 0 ) {
- long queueSize = mQueue.size();
-
- LOG.info("AsyncAuditProvider-stats:" + mName + ": past " + formatIntervalForLog(intervalSinceLastLog)
- + ": inLogs=" + intervalInLogCount.get()
- + ", outLogs=" + intervalOutLogCount.get()
- + ", dropped=" + intervalDropCount.get()
- + ", currentQueueSize=" + queueSize);
-
- LOG.info("AsyncAuditProvider-stats:" + mName + ": process lifetime"
- + ": inLogs=" + lifeTimeInLogCount.get()
- + ", outLogs=" + lifeTimeOutLogCount.get()
- + ", dropped=" + lifeTimeDropCount.get());
- }
-
- lastIntervalLogTime = System.currentTimeMillis();
- intervalInLogCount.set(0);
- intervalOutLogCount.set(0);
- intervalDropCount.set(0);
- }
- }
-
- private boolean isEmpty() {
- return mQueue.isEmpty();
- }
-
- public void waitToComplete(long maxWaitSeconds) {
- LOG.debug("==> AsyncAuditProvider.waitToComplete()");
-
- try {
- for (long waitTime = 0; !isEmpty()
- && (maxWaitSeconds <= 0 || maxWaitSeconds > waitTime); waitTime += mWaitToCompleteLoopIntervalSecs) {
- try {
- LOG.info(String.format("%d messages yet to be flushed by %s. Sleeoping for %d sec", mQueue.size(), mName, mWaitToCompleteLoopIntervalSecs));
- Thread.sleep(mWaitToCompleteLoopIntervalSecs * 1000L);
- } catch (InterruptedException excp) {
- // someone really wants service to exit, abandon unwritten audits and exit.
- LOG.warn("Caught interrupted exception! " + mQueue.size() + " messages still unflushed! Won't wait for queue to flush, exiting...", excp);
- break;
- }
- }
- } finally {
- LOG.debug("<== AsyncAuditProvider.waitToComplete()");
- }
- }
-
- private long getTimeTillNextFlush() {
- long timeTillNextFlush = mMaxFlushInterval;
-
- if (mMaxFlushInterval > 0) {
-
- if (lastFlushTime != 0) {
- long timeSinceLastFlush = System.currentTimeMillis()
- - lastFlushTime;
-
- if (timeSinceLastFlush >= mMaxFlushInterval) {
- timeTillNextFlush = 0;
- } else {
- timeTillNextFlush = mMaxFlushInterval - timeSinceLastFlush;
- }
- }
- }
-
- return timeTillNextFlush;
- }
+ return timeTillNextFlush;
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditFileCacheProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditFileCacheProvider.java
index 7957f2e10..9d6894f75 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditFileCacheProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditFileCacheProvider.java
@@ -31,49 +31,60 @@ import java.util.Properties;
*/
public class AuditFileCacheProvider extends BaseAuditHandler {
- private static final Logger logger = LoggerFactory.getLogger(AuditFileCacheProvider.class);
+ private static final Logger logger = LoggerFactory.getLogger(AuditFileCacheProvider.class);
- AuditFileCacheProviderSpool fileSpooler = null;
- AuditHandler consumer = null;
+ AuditFileCacheProviderSpool fileSpooler;
+ AuditHandler consumer;
AuditFileCacheProvider(AuditHandler consumer) {
this.consumer = consumer;
}
+ @Override
+ public boolean log(AuditEventBase event) {
+ boolean ret = false;
+
+ if (event != null) {
+ fileSpooler.stashLogs(event);
+
+ if (fileSpooler.isSpoolingSuccessful()) {
+ ret = true;
+ }
+ }
+
+ return ret;
+ }
+
public void init(Properties prop, String basePropertyName) {
String propPrefix = "xasecure.audit.filecache";
+
if (basePropertyName != null) {
propPrefix = basePropertyName;
}
+
super.init(prop, propPrefix);
+
//init Consumer
if (consumer != null) {
consumer.init(prop, propPrefix);
}
+
//init AuditFileCacheSpooler
fileSpooler = new AuditFileCacheProviderSpool(consumer);
- fileSpooler.init(prop,propPrefix);
- }
- @Override
- public boolean log(AuditEventBase event) {
- boolean ret = false;
- if ( event != null) {
- fileSpooler.stashLogs(event);
- if ( fileSpooler.isSpoolingSuccessful()) {
- ret = true;
- }
- }
- return ret;
+
+ fileSpooler.init(prop, propPrefix);
}
@Override
public boolean log(Collection<AuditEventBase> events) {
boolean ret = true;
- if ( events != null) {
+
+ if (events != null) {
for (AuditEventBase event : events) {
ret = log(event);
}
}
+
return ret;
}
@@ -83,6 +94,7 @@ public class AuditFileCacheProvider extends BaseAuditHandler {
if (consumer != null) {
consumer.start();
}
+
if (fileSpooler != null) {
// start AuditFileSpool thread
fileSpooler.start();
@@ -91,7 +103,8 @@ public class AuditFileCacheProvider extends BaseAuditHandler {
@Override
public void stop() {
- logger.info("Stop called. name=" + getName());
+ logger.info("stop() called. name={}", getName());
+
if (consumer != null) {
consumer.stop();
}
@@ -99,24 +112,27 @@ public class AuditFileCacheProvider extends BaseAuditHandler {
@Override
public void waitToComplete() {
- logger.info("waitToComplete called. name=" + getName());
- if ( consumer != null) {
+ logger.info("waitToComplete() called. name={}", getName());
+
+ if (consumer != null) {
consumer.waitToComplete();
}
}
@Override
public void waitToComplete(long timeout) {
- logger.info("waitToComplete called. name=" + getName());
- if ( consumer != null) {
+ logger.info("waitToComplete(timeout={}) called. name={}", timeout, getName());
+
+ if (consumer != null) {
consumer.waitToComplete(timeout);
}
}
@Override
public void flush() {
- logger.info("waitToComplete. name=" + getName());
- if ( consumer != null) {
+ logger.info("flush() called. name={}", getName());
+
+ if (consumer != null) {
consumer.flush();
}
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java
index dd02255fc..8c51539d5 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditHandler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,25 +18,33 @@
package org.apache.ranger.audit.provider;
+import org.apache.ranger.audit.model.AuditEventBase;
+
import java.io.File;
import java.util.Collection;
import java.util.Properties;
-import org.apache.ranger.audit.model.AuditEventBase;
-
public interface AuditHandler {
- boolean log(AuditEventBase event);
- boolean log(Collection<AuditEventBase> events);
+ boolean log(AuditEventBase event);
+
+ boolean log(Collection<AuditEventBase> events);
+
+ boolean logJSON(String event);
+
+ boolean logJSON(Collection<String> events);
- boolean logJSON(String event);
- boolean logJSON(Collection<String> events);
boolean logFile(File file);
void init(Properties prop);
+
void init(Properties prop, String basePropertyName);
+
void start();
+
void stop();
+
void waitToComplete();
+
void waitToComplete(long timeout);
/**
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditMessageException.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditMessageException.java
index 3ff04eee6..a396d37a1 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditMessageException.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditMessageException.java
@@ -25,43 +25,41 @@ package org.apache.ranger.audit.provider;
* a transient error
*/
public class AuditMessageException extends Exception {
+ private static final long serialVersionUID = 1L;
- private static final long serialVersionUID = 1L;
+ public AuditMessageException() {
+ }
- public AuditMessageException() {
- }
+ /**
+ * @param message
+ */
+ public AuditMessageException(String message) {
+ super(message);
+ }
- /**
- * @param message
- */
- public AuditMessageException(String message) {
- super(message);
- }
+ /**
+ * @param cause
+ */
+ public AuditMessageException(Throwable cause) {
+ super(cause);
+ }
- /**
- * @param cause
- */
- public AuditMessageException(Throwable cause) {
- super(cause);
- }
-
- /**
- * @param message
- * @param cause
- */
- public AuditMessageException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * @param message
- * @param cause
- * @param enableSuppression
- * @param writableStackTrace
- */
- public AuditMessageException(String message, Throwable cause,
- boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
+ /**
+ * @param message
+ * @param cause
+ */
+ public AuditMessageException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ /**
+ * @param message
+ * @param cause
+ * @param enableSuppression
+ * @param writableStackTrace
+ */
+ public AuditMessageException(String message, Throwable cause,
+ boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index c10dd9ffa..ed38fad2c 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -18,16 +18,13 @@
package org.apache.ranger.audit.provider;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.ranger.audit.destination.*;
+import org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination;
+import org.apache.ranger.audit.destination.ElasticSearchAuditDestination;
+import org.apache.ranger.audit.destination.FileAuditDestination;
+import org.apache.ranger.audit.destination.HDFSAuditDestination;
+import org.apache.ranger.audit.destination.Log4JAuditDestination;
+import org.apache.ranger.audit.destination.SolrAuditDestination;
import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider;
import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider;
import org.apache.ranger.audit.provider.solr.SolrAuditProvider;
@@ -39,6 +36,14 @@ import org.apache.ranger.audit.queue.AuditSummaryQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
/*
* TODO:
* 1) Flag to enable/disable audit logging
@@ -47,511 +52,524 @@ import org.slf4j.LoggerFactory;
*/
public class AuditProviderFactory {
- private static final Logger LOG = LoggerFactory
- .getLogger(AuditProviderFactory.class);
-
- public static final String AUDIT_IS_ENABLED_PROP = "xasecure.audit.is.enabled";
- public static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
- public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled";
- public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
- public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
-
- public static final String AUDIT_DEST_BASE = "xasecure.audit.destination";
- public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds";
- public static final String AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP = "xasecure.audit.provider.filecache.is.enabled";
- public static final String FILE_QUEUE_TYPE = "filequeue";
- public static final String DEFAULT_QUEUE_TYPE = "memoryqueue";
- public static final int AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT = 30;
-
- public static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024;
- public static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000;
-
- private static final int RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY = 30;
-
- private volatile static AuditProviderFactory sFactory = null;
-
- private AuditHandler mProvider = null;
- private String componentAppType = "";
- private boolean mInitDone = false;
- private JVMShutdownHook jvmShutdownHook = null;
- private ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster","hbaseRegional"));
-
- public AuditProviderFactory() {
- LOG.info("AuditProviderFactory: creating..");
-
- mProvider = getDefaultProvider();
- }
-
- public static AuditProviderFactory getInstance() {
- AuditProviderFactory ret = sFactory;
- if(ret == null) {
- synchronized(AuditProviderFactory.class) {
- ret = sFactory;
- if(ret == null) {
- ret = sFactory = new AuditProviderFactory();
- }
- }
- }
-
- return ret;
- }
-
- public AuditHandler getAuditProvider() {
- return mProvider;
- }
-
- public boolean isInitDone() {
- return mInitDone;
- }
-
- /**
- * call shutdown hook to provide a way to
- * shutdown gracefully in addition to the ShutdownHook mechanism
- */
- public void shutdown() {
- if (isInitDone() && jvmShutdownHook != null) {
- jvmShutdownHook.run();
- }
- }
-
- public synchronized void init(Properties props, String appType) {
- LOG.info("AuditProviderFactory: initializing..");
-
- if (mInitDone) {
- LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
- }
- mInitDone = true;
- componentAppType = appType;
- MiscUtil.setApplicationType(appType);
-
- boolean isEnabled = MiscUtil.getBooleanProperty(props,
- AUDIT_IS_ENABLED_PROP, true);
+ private static final Logger LOG = LoggerFactory.getLogger(AuditProviderFactory.class);
+
+ public static final String AUDIT_IS_ENABLED_PROP = "xasecure.audit.is.enabled";
+ public static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
+ public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled";
+ public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
+ public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
+ public static final String AUDIT_DEST_BASE = "xasecure.audit.destination";
+ public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds";
+ public static final String AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP = "xasecure.audit.provider.filecache.is.enabled";
+ public static final String FILE_QUEUE_TYPE = "filequeue";
+ public static final String DEFAULT_QUEUE_TYPE = "memoryqueue";
+ public static final int AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT = 30;
+ public static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024;
+ public static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000;
+
+ private static final int RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY = 30;
+
+ private static volatile AuditProviderFactory sFactory;
+
+ private AuditHandler mProvider;
+ private String componentAppType = "";
+ private boolean mInitDone;
+ private JVMShutdownHook jvmShutdownHook;
+ private final ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional"));
+
+ public AuditProviderFactory() {
+ LOG.info("AuditProviderFactory: creating..");
+
+ mProvider = getDefaultProvider();
+ }
+
+ public static AuditProviderFactory getInstance() {
+ AuditProviderFactory ret = sFactory;
+
+ if (ret == null) {
+ synchronized (AuditProviderFactory.class) {
+ ret = sFactory;
+
+ if (ret == null) {
+ ret = new AuditProviderFactory();
+
+ sFactory = ret;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ public AuditHandler getAuditProvider() {
+ return mProvider;
+ }
+
+ public boolean isInitDone() {
+ return mInitDone;
+ }
+
+ /**
+ * call shutdown hook to provide a way to
+ * shutdown gracefully in addition to the ShutdownHook mechanism
+ */
+ public void shutdown() {
+ if (isInitDone() && jvmShutdownHook != null) {
+ jvmShutdownHook.run();
+ }
+ }
+
+ public synchronized void init(Properties props, String appType) {
+ LOG.info("AuditProviderFactory: initializing..");
+
+ if (mInitDone) {
+ LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
+ }
+
+ mInitDone = true;
+ componentAppType = appType;
+
+ MiscUtil.setApplicationType(appType);
+
+ boolean isEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_ENABLED_PROP, true);
+
if (!isEnabled) {
LOG.info("AuditProviderFactory: Audit not enabled..");
+
mProvider = getDefaultProvider();
+
return;
}
- boolean isAuditToHdfsEnabled = MiscUtil.getBooleanProperty(props,
- AUDIT_HDFS_IS_ENABLED_PROP, false);
- boolean isAuditToLog4jEnabled = MiscUtil.getBooleanProperty(props,
- AUDIT_LOG4J_IS_ENABLED_PROP, false);
- boolean isAuditToKafkaEnabled = MiscUtil.getBooleanProperty(props,
- AUDIT_KAFKA_IS_ENABLED_PROP, false);
- boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props,
- AUDIT_SOLR_IS_ENABLED_PROP, false);
-
- boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props,
- AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false);
-
- List<AuditHandler> providers = new ArrayList<AuditHandler>();
-
- for (Object propNameObj : props.keySet()) {
- LOG.info("AUDIT PROPERTY: " + propNameObj.toString() + "="
- + props.getProperty(propNameObj.toString()));
- }
-
- // Process new audit configurations
- List<String> destNameList = new ArrayList<String>();
-
- for (Object propNameObj : props.keySet()) {
- String propName = propNameObj.toString();
- if (!propName.startsWith(AUDIT_DEST_BASE)) {
- continue;
- }
- String destName = propName.substring(AUDIT_DEST_BASE.length() + 1);
- List<String> splits = MiscUtil.toArray(destName, ".");
- if (splits.size() > 1) {
- continue;
- }
- String value = props.getProperty(propName);
- if (value.equalsIgnoreCase("enable")
- || value.equalsIgnoreCase("enabled")
- || value.equalsIgnoreCase("true")) {
- destNameList.add(destName);
- LOG.info("Audit destination " + propName + " is set to "
- + value);
- }
- }
-
- for (String destName : destNameList) {
- String destPropPrefix = AUDIT_DEST_BASE + "." + destName;
- AuditHandler destProvider = getProviderFromConfig(props,
- destPropPrefix, destName, null);
-
- if (destProvider != null) {
- destProvider.init(props, destPropPrefix);
-
- String queueName = MiscUtil.getStringProperty(props,
- destPropPrefix + "." + AuditQueue.PROP_QUEUE);
- if (queueName == null || queueName.isEmpty()) {
- LOG.info(destPropPrefix + "." + AuditQueue.PROP_QUEUE
- + " is not set. Setting queue to batch for "
- + destName);
- queueName = "batch";
- }
- LOG.info("queue for " + destName + " is " + queueName);
- if (queueName != null && !queueName.isEmpty()
- && !queueName.equalsIgnoreCase("none")) {
- String queuePropPrefix = destPropPrefix + "." + queueName;
- AuditHandler queueProvider = getProviderFromConfig(props,
- queuePropPrefix, queueName, destProvider);
- if (queueProvider != null) {
- if (queueProvider instanceof AuditQueue) {
- AuditQueue qProvider = (AuditQueue) queueProvider;
- qProvider.init(props, queuePropPrefix);
- providers.add(queueProvider);
- } else {
- LOG.error("Provider queue doesn't extend AuditQueue. Destination="
- + destName
- + " can't be created. queueName="
- + queueName);
- }
- } else {
- LOG.error("Queue provider for destination " + destName
- + " can't be created. queueName=" + queueName);
- }
- } else {
- LOG.info("Audit destination " + destProvider.getName()
- + " added to provider list");
- providers.add(destProvider);
- }
- }
- }
- if (providers.size() > 0) {
- LOG.info("Using v3 audit configuration");
- AuditHandler consumer = providers.get(0);
-
- // Possible pipeline is:
- // async_queue -> summary_queue -> multidestination -> batch_queue
- // -> hdfs_destination
- // -> batch_queue -> solr_destination
- // -> batch_queue -> kafka_destination
- // Above, up to multidestination, the providers are same, then it
- // branches out in parallel.
-
- // Set the providers in the reverse order e.g.
-
- if (providers.size() > 1) {
- // If there are more than one destination, then we need multi
- // destination to process it in parallel
- LOG.info("MultiDestAuditProvider is used. Destination count="
- + providers.size());
- MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
- multiDestProvider.init(props);
- multiDestProvider.addAuditProviders(providers);
- consumer = multiDestProvider;
- }
-
- // Let's see if Summary is enabled, then summarize before sending it
- // downstream
- String propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX;
- boolean summaryEnabled = MiscUtil.getBooleanProperty(props,
- propPrefix + "." + "summary" + "." + "enabled", false);
- AuditSummaryQueue summaryQueue = null;
- if (summaryEnabled) {
- LOG.info("AuditSummaryQueue is enabled");
- summaryQueue = new AuditSummaryQueue(consumer);
- summaryQueue.init(props, propPrefix);
- consumer = summaryQueue;
- } else {
- LOG.info("AuditSummaryQueue is disabled");
- }
-
- if (!isAuditFileCacheProviderEnabled) {
- // Create the AsysnQueue
- AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer);
- propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async";
- asyncQueue.init(props, propPrefix);
- asyncQueue.setParentPath(componentAppType);
- mProvider = asyncQueue;
- LOG.info("Starting audit queue " + mProvider.getName());
- mProvider.start();
- } else {
- // Assign AsyncQueue to AuditFileCacheProvider
- AuditFileCacheProvider auditFileCacheProvider = new AuditFileCacheProvider(consumer);
- propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "filecache";
- auditFileCacheProvider.init(props, propPrefix);
- auditFileCacheProvider.setParentPath(componentAppType);
- mProvider = auditFileCacheProvider;
- LOG.info("Starting Audit File Cache Provider " + mProvider.getName());
- mProvider.start();
- }
- } else {
- LOG.info("No v3 audit configuration found. Trying v2 audit configurations");
- if (!isEnabled
- || !(isAuditToHdfsEnabled
- || isAuditToKafkaEnabled || isAuditToLog4jEnabled
- || isAuditToSolrEnabled || providers.size() == 0)) {
- LOG.info("AuditProviderFactory: Audit not enabled..");
-
- mProvider = getDefaultProvider();
-
- return;
- }
-
- if (isAuditToHdfsEnabled) {
- LOG.info("HdfsAuditProvider is enabled");
-
- HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
-
- boolean isAuditToHdfsAsync = MiscUtil.getBooleanProperty(props,
- HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
-
- if (isAuditToHdfsAsync) {
- int maxQueueSize = MiscUtil.getIntProperty(props,
- HdfsAuditProvider.AUDIT_HDFS_MAX_QUEUE_SIZE_PROP,
- AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
- int maxFlushInterval = MiscUtil
- .getIntProperty(
- props,
- HdfsAuditProvider.AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP,
- AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
-
- AsyncAuditProvider asyncProvider = new AsyncAuditProvider(
- "HdfsAuditProvider", maxQueueSize,
- maxFlushInterval, hdfsProvider);
-
- providers.add(asyncProvider);
- } else {
- providers.add(hdfsProvider);
- }
- }
-
- if (isAuditToKafkaEnabled) {
- LOG.info("KafkaAuditProvider is enabled");
- KafkaAuditProvider kafkaProvider = new KafkaAuditProvider();
- kafkaProvider.init(props);
-
- if (kafkaProvider.isAsync()) {
- AsyncAuditProvider asyncProvider = new AsyncAuditProvider(
- "MyKafkaAuditProvider", 1000, 1000, kafkaProvider);
- providers.add(asyncProvider);
- } else {
- providers.add(kafkaProvider);
- }
- }
-
- if (isAuditToSolrEnabled) {
- LOG.info("SolrAuditProvider is enabled");
- SolrAuditProvider solrProvider = new SolrAuditProvider();
- solrProvider.init(props);
-
- if (solrProvider.isAsync()) {
- AsyncAuditProvider asyncProvider = new AsyncAuditProvider(
- "MySolrAuditProvider", 1000, 1000, solrProvider);
- providers.add(asyncProvider);
- } else {
- providers.add(solrProvider);
- }
- }
-
- if (isAuditToLog4jEnabled) {
- Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
-
- boolean isAuditToLog4jAsync = MiscUtil.getBooleanProperty(
- props, Log4jAuditProvider.AUDIT_LOG4J_IS_ASYNC_PROP,
- false);
-
- if (isAuditToLog4jAsync) {
- int maxQueueSize = MiscUtil.getIntProperty(props,
- Log4jAuditProvider.AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP,
- AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
- int maxFlushInterval = MiscUtil
- .getIntProperty(
- props,
- Log4jAuditProvider.AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP,
- AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
-
- AsyncAuditProvider asyncProvider = new AsyncAuditProvider(
- "Log4jAuditProvider", maxQueueSize,
- maxFlushInterval, log4jProvider);
-
- providers.add(asyncProvider);
- } else {
- providers.add(log4jProvider);
- }
- }
- if (providers.size() == 0) {
- mProvider = getDefaultProvider();
- } else if (providers.size() == 1) {
- mProvider = providers.get(0);
- } else {
- MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
-
- multiDestProvider.addAuditProviders(providers);
-
- mProvider = multiDestProvider;
- }
-
- mProvider.init(props);
- mProvider.start();
- }
-
- installJvmShutdownHook(props);
- }
-
- private AuditHandler getProviderFromConfig(Properties props,
- String propPrefix, String providerName, AuditHandler consumer) {
- AuditHandler provider = null;
- String className = MiscUtil.getStringProperty(props, propPrefix + "."
- + BaseAuditHandler.PROP_CLASS_NAME);
- if (className != null && !className.isEmpty()) {
- try {
- Class<?> handlerClass = Class.forName(className);
- if (handlerClass.isAssignableFrom(AuditQueue.class)) {
- // Queue class needs consumer
- handlerClass.getDeclaredConstructor(AuditHandler.class)
- .newInstance(consumer);
- } else {
- provider = (AuditHandler) Class.forName(className)
- .newInstance();
- }
- } catch (Exception e) {
- LOG.error("Can't instantiate audit class for providerName="
- + providerName + ", className=" + className
- + ", propertyPrefix=" + propPrefix, e);
- }
- } else {
- if (providerName.equalsIgnoreCase("file")) {
- provider = new FileAuditDestination();
- } else if (providerName.equalsIgnoreCase("hdfs")) {
- provider = new HDFSAuditDestination();
- } else if (providerName.equalsIgnoreCase("solr")) {
- provider = new SolrAuditDestination();
- } else if (providerName.equalsIgnoreCase("elasticsearch")) {
- provider = new ElasticSearchAuditDestination();
- } else if (providerName.equalsIgnoreCase("amazon_cloudwatch")) {
- provider = new AmazonCloudWatchAuditDestination();
- } else if (providerName.equalsIgnoreCase("kafka")) {
- provider = new KafkaAuditProvider();
- } else if (providerName.equalsIgnoreCase("log4j")) {
- provider = new Log4JAuditDestination();
- } else if (providerName.equalsIgnoreCase("batch")) {
- provider = getAuditProvider(props, propPrefix, consumer);
- } else if (providerName.equalsIgnoreCase("async")) {
- provider = new AuditAsyncQueue(consumer);
- } else {
- LOG.error("Provider name doesn't have any class associated with it. providerName="
- + providerName + ", propertyPrefix=" + propPrefix);
- }
- }
- if (provider != null && provider instanceof AuditQueue) {
- if (consumer == null) {
- LOG.error("consumer can't be null for AuditQueue. queue="
- + provider.getName() + ", propertyPrefix=" + propPrefix);
- provider = null;
- }
- }
- return provider;
- }
-
- private AuditHandler getAuditProvider(Properties props, String propPrefix, AuditHandler consumer) {
- AuditHandler ret = null;
- String queueType = MiscUtil.getStringProperty(props, propPrefix + "." + "queuetype", DEFAULT_QUEUE_TYPE);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> AuditProviderFactory.getAuditProvider() propPerfix= " + propPrefix + ", " + " queueType= " + queueType);
- }
-
- if (FILE_QUEUE_TYPE.equalsIgnoreCase(queueType)) {
- AuditFileQueue auditFileQueue = new AuditFileQueue(consumer);
- String propPrefixFileQueue = propPrefix + "." + FILE_QUEUE_TYPE;
- auditFileQueue.init(props, propPrefixFileQueue);
- ret = new AuditBatchQueue(auditFileQueue);
- } else {
- ret = new AuditBatchQueue(consumer);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== AuditProviderFactory.getAuditProvider()");
- }
-
- return ret;
- }
-
- private AuditHandler getDefaultProvider() {
- return new DummyAuditProvider();
- }
-
- private void installJvmShutdownHook(Properties props) {
- int shutdownHookMaxWaitSeconds = MiscUtil.getIntProperty(props, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT);
- jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds);
- String appType = this.componentAppType;
- if (appType != null && !hbaseAppTypes.contains(appType)) {
- ShutdownHookManager.get().addShutdownHook(jvmShutdownHook, RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY);
- }
- }
-
- private static class RangerAsyncAuditCleanup implements Runnable {
-
- final Semaphore startCleanup;
- final Semaphore doneCleanup;
- final AuditHandler mProvider;
-
- RangerAsyncAuditCleanup(AuditHandler provider, Semaphore startCleanup, Semaphore doneCleanup) {
- this.startCleanup = startCleanup;
- this.doneCleanup = doneCleanup;
- this.mProvider = provider;
- }
-
- @Override
- public void run() {
- while (true) {
- LOG.info("RangerAsyncAuditCleanup: Waiting to audit cleanup start signal");
- try {
- startCleanup.acquire();
- } catch (InterruptedException e) {
- LOG.error("RangerAsyncAuditCleanup: Interrupted while waiting for audit startCleanup signal! Exiting the thread...", e);
- break;
- }
- LOG.info("RangerAsyncAuditCleanup: Starting cleanup");
- mProvider.waitToComplete();
- mProvider.stop();
- doneCleanup.release();
- LOG.info("RangerAsyncAuditCleanup: Done cleanup");
- }
- }
- }
-
- private static class JVMShutdownHook extends Thread {
- final Semaphore startCleanup = new Semaphore(0);
- final Semaphore doneCleanup = new Semaphore(0);
- final Thread cleanupThread;
- final int maxWait;
- final AtomicBoolean done = new AtomicBoolean(false);
-
- public JVMShutdownHook(AuditHandler provider, int maxWait) {
- this.maxWait = maxWait;
- Runnable runnable = new RangerAsyncAuditCleanup(provider, startCleanup, doneCleanup);
- cleanupThread = new Thread(runnable, "Ranger async Audit cleanup");
- cleanupThread.setDaemon(true);
- cleanupThread.start();
- }
-
- public void run() {
- if (!done.compareAndSet(false, true)) {
- LOG.info("==> JVMShutdownHook.run() already done by another thread");
- return;
- }
- LOG.info("==> JVMShutdownHook.run()");
- LOG.info("JVMShutdownHook: Signalling async audit cleanup to start.");
- startCleanup.release();
- try {
- Long start = System.currentTimeMillis();
- LOG.info("JVMShutdownHook: Waiting up to " + maxWait + " seconds for audit cleanup to finish.");
- boolean cleanupFinishedInTime = doneCleanup.tryAcquire(maxWait, TimeUnit.SECONDS);
- if (cleanupFinishedInTime) {
- LOG.info("JVMShutdownHook: Audit cleanup finished after " + (System.currentTimeMillis() - start) + " milli seconds");
- } else {
- LOG.warn("JVMShutdownHook: could not detect finishing of audit cleanup even after waiting for " + maxWait + " seconds!");
- }
- } catch (InterruptedException e) {
- LOG.error("JVMShutdownHook: Interrupted while waiting for completion of Async executor!", e);
- }
- LOG.info("JVMShutdownHook: Interrupting ranger async audit cleanup thread");
- cleanupThread.interrupt();
- LOG.info("<== JVMShutdownHook.run()");
- }
- }
+ boolean isAuditToHdfsEnabled = MiscUtil.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
+ boolean isAuditToLog4jEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
+ boolean isAuditToKafkaEnabled = MiscUtil.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false);
+ boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false);
+ boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props, AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false);
+
+ List<AuditHandler> providers = new ArrayList<>();
+
+ for (Object propNameObj : props.keySet()) {
+ LOG.info("AUDIT PROPERTY: {}={}", propNameObj, props.getProperty(propNameObj.toString()));
+ }
+
+ // Process new audit configurations
+ List<String> destNameList = new ArrayList<>();
+
+ for (Object propNameObj : props.keySet()) {
+ String propName = propNameObj.toString();
+
+ if (!propName.startsWith(AUDIT_DEST_BASE)) {
+ continue;
+ }
+
+ String destName = propName.substring(AUDIT_DEST_BASE.length() + 1);
+ List<String> splits = MiscUtil.toArray(destName, ".");
+
+ if (splits.size() > 1) {
+ continue;
+ }
+
+ String value = props.getProperty(propName);
+
+ if (value.equalsIgnoreCase("enable") || value.equalsIgnoreCase("enabled") || value.equalsIgnoreCase("true")) {
+ destNameList.add(destName);
+
+ LOG.info("Audit destination {} is set to {}", propName, value);
+ }
+ }
+
+ for (String destName : destNameList) {
+ String destPropPrefix = AUDIT_DEST_BASE + "." + destName;
+ AuditHandler destProvider = getProviderFromConfig(props, destPropPrefix, destName, null);
+
+ if (destProvider != null) {
+ destProvider.init(props, destPropPrefix);
+
+ String queueName = MiscUtil.getStringProperty(props, destPropPrefix + "." + AuditQueue.PROP_QUEUE);
+
+ if (queueName == null || queueName.isEmpty()) {
+ LOG.info("{}.{} is not set. Setting queue to batch for {}", destPropPrefix, AuditQueue.PROP_QUEUE, destName);
+
+ queueName = "batch";
+ }
+
+ LOG.info("queue for {} is {}", destName, queueName);
+
+ if (queueName != null && !queueName.isEmpty() && !queueName.equalsIgnoreCase("none")) {
+ String queuePropPrefix = destPropPrefix + "." + queueName;
+ AuditHandler queueProvider = getProviderFromConfig(props, queuePropPrefix, queueName, destProvider);
+
+ if (queueProvider != null) {
+ if (queueProvider instanceof AuditQueue) {
+ AuditQueue qProvider = (AuditQueue) queueProvider;
+
+ qProvider.init(props, queuePropPrefix);
+
+ providers.add(queueProvider);
+ } else {
+ LOG.error("Provider queue doesn't extend AuditQueue. Destination={} can't be created. queueName={}", destName, queueName);
+ }
+ } else {
+ LOG.error("Queue provider for destination {} can't be created. queueName={}", destName, queueName);
+ }
+ } else {
+ LOG.info("Audit destination {} added to provider list", destProvider.getName());
+
+ providers.add(destProvider);
+ }
+ }
+ }
+ if (!providers.isEmpty()) {
+ LOG.info("Using v3 audit configuration");
+
+ AuditHandler consumer = providers.get(0);
+
+ // Possible pipeline is:
+ // async_queue -> summary_queue -> multidestination -> batch_queue
+ // -> hdfs_destination
+ // -> batch_queue -> solr_destination
+ // -> batch_queue -> kafka_destination
+ // Above, up to multidestination, the providers are same, then it
+ // branches out in parallel.
+
+ // Set the providers in the reverse order e.g.
+
+ if (providers.size() > 1) {
+ // If there are more than one destination, then we need multi destination to process it in parallel
+ LOG.info("MultiDestAuditProvider is used. Destination count={}", providers.size());
+
+ MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
+
+ multiDestProvider.init(props);
+ multiDestProvider.addAuditProviders(providers);
+
+ consumer = multiDestProvider;
+ }
+
+ // Let's see if Summary is enabled, then summarize before sending it downstream
+ String propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX;
+ boolean summaryEnabled = MiscUtil.getBooleanProperty(props, propPrefix + "." + "summary" + "." + "enabled", false);
+
+ if (summaryEnabled) {
+ LOG.info("AuditSummaryQueue is enabled");
+
+ AuditSummaryQueue summaryQueue = new AuditSummaryQueue(consumer);
+
+ summaryQueue.init(props, propPrefix);
+
+ consumer = summaryQueue;
+ } else {
+ LOG.info("AuditSummaryQueue is disabled");
+ }
+
+ if (!isAuditFileCacheProviderEnabled) {
+ // Create the AsysnQueue
+ AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer);
+
+ propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async";
+
+ asyncQueue.init(props, propPrefix);
+ asyncQueue.setParentPath(componentAppType);
+
+ mProvider = asyncQueue;
+
+ LOG.info("Starting audit queue {}", mProvider.getName());
+
+ mProvider.start();
+ } else {
+ // Assign AsyncQueue to AuditFileCacheProvider
+ AuditFileCacheProvider auditFileCacheProvider = new AuditFileCacheProvider(consumer);
+
+ propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "filecache";
+
+ auditFileCacheProvider.init(props, propPrefix);
+ auditFileCacheProvider.setParentPath(componentAppType);
+
+ mProvider = auditFileCacheProvider;
+
+ LOG.info("Starting Audit File Cache Provider {}", mProvider.getName());
+
+ mProvider.start();
+ }
+ } else {
+ LOG.info("No v3 audit configuration found. Trying v2 audit configurations");
+
+ if (!isEnabled || !(isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled || providers.isEmpty())) {
+ LOG.info("AuditProviderFactory: Audit not enabled..");
+
+ mProvider = getDefaultProvider();
+
+ return;
+ }
+
+ if (isAuditToHdfsEnabled) {
+ LOG.info("HdfsAuditProvider is enabled");
+
+ HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
+
+ boolean isAuditToHdfsAsync = MiscUtil.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
+
+ if (isAuditToHdfsAsync) {
+ int maxQueueSize = MiscUtil.getIntProperty(props, HdfsAuditProvider.AUDIT_HDFS_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
+ int maxFlushInterval = MiscUtil.getIntProperty(props, HdfsAuditProvider.AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
+
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider("HdfsAuditProvider", maxQueueSize, maxFlushInterval, hdfsProvider);
+
+ providers.add(asyncProvider);
+ } else {
+ providers.add(hdfsProvider);
+ }
+ }
+
+ if (isAuditToKafkaEnabled) {
+ LOG.info("KafkaAuditProvider is enabled");
+
+ KafkaAuditProvider kafkaProvider = new KafkaAuditProvider();
+
+ kafkaProvider.init(props);
+
+ if (kafkaProvider.isAsync()) {
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MyKafkaAuditProvider", 1000, 1000, kafkaProvider);
+
+ providers.add(asyncProvider);
+ } else {
+ providers.add(kafkaProvider);
+ }
+ }
+
+ if (isAuditToSolrEnabled) {
+ LOG.info("SolrAuditProvider is enabled");
+
+ SolrAuditProvider solrProvider = new SolrAuditProvider();
+
+ solrProvider.init(props);
+
+ if (solrProvider.isAsync()) {
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", 1000, 1000, solrProvider);
+
+ providers.add(asyncProvider);
+ } else {
+ providers.add(solrProvider);
+ }
+ }
+
+ if (isAuditToLog4jEnabled) {
+ Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
+
+ boolean isAuditToLog4jAsync = MiscUtil.getBooleanProperty(props, Log4jAuditProvider.AUDIT_LOG4J_IS_ASYNC_PROP, false);
+
+ if (isAuditToLog4jAsync) {
+ int maxQueueSize = MiscUtil.getIntProperty(props, Log4jAuditProvider.AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT);
+ int maxFlushInterval = MiscUtil.getIntProperty(props, Log4jAuditProvider.AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP, AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT);
+
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider("Log4jAuditProvider", maxQueueSize, maxFlushInterval, log4jProvider);
+
+ providers.add(asyncProvider);
+ } else {
+ providers.add(log4jProvider);
+ }
+ }
+ if (providers.isEmpty()) {
+ mProvider = getDefaultProvider();
+ } else if (providers.size() == 1) {
+ mProvider = providers.get(0);
+ } else {
+ MultiDestAuditProvider multiDestProvider = new MultiDestAuditProvider();
+
+ multiDestProvider.addAuditProviders(providers);
+
+ mProvider = multiDestProvider;
+ }
+
+ mProvider.init(props);
+ mProvider.start();
+ }
+
+ installJvmShutdownHook(props);
+ }
+
+ private AuditHandler getProviderFromConfig(Properties props, String propPrefix, String providerName, AuditHandler consumer) {
+ AuditHandler provider = null;
+ String className = MiscUtil.getStringProperty(props, propPrefix + "." + BaseAuditHandler.PROP_CLASS_NAME);
+
+ if (className != null && !className.isEmpty()) {
+ try {
+ Class<?> handlerClass = Class.forName(className);
+
+ if (handlerClass.isAssignableFrom(AuditQueue.class)) {
+ // Queue class needs consumer
+ handlerClass.getDeclaredConstructor(AuditHandler.class).newInstance(consumer);
+ } else {
+ provider = (AuditHandler) Class.forName(className).newInstance();
+ }
+ } catch (Exception e) {
+ LOG.error("Can't instantiate audit class for providerName={}, className={}, propertyPrefix={}", providerName, className, propPrefix, e);
+ }
+ } else {
+ if (providerName.equalsIgnoreCase("file")) {
+ provider = new FileAuditDestination();
+ } else if (providerName.equalsIgnoreCase("hdfs")) {
+ provider = new HDFSAuditDestination();
+ } else if (providerName.equalsIgnoreCase("solr")) {
+ provider = new SolrAuditDestination();
+ } else if (providerName.equalsIgnoreCase("elasticsearch")) {
+ provider = new ElasticSearchAuditDestination();
+ } else if (providerName.equalsIgnoreCase("amazon_cloudwatch")) {
+ provider = new AmazonCloudWatchAuditDestination();
+ } else if (providerName.equalsIgnoreCase("kafka")) {
+ provider = new KafkaAuditProvider();
+ } else if (providerName.equalsIgnoreCase("log4j")) {
+ provider = new Log4JAuditDestination();
+ } else if (providerName.equalsIgnoreCase("batch")) {
+ provider = getAuditProvider(props, propPrefix, consumer);
+ } else if (providerName.equalsIgnoreCase("async")) {
+ provider = new AuditAsyncQueue(consumer);
+ } else {
+ LOG.error("Provider name doesn't have any class associated with it. providerName={}, propertyPrefix={}", providerName, propPrefix);
+ }
+ }
+
+ if (provider instanceof AuditQueue) {
+ if (consumer == null) {
+ LOG.error("consumer can't be null for AuditQueue. queue={}, propertyPrefix={}", provider.getName(), propPrefix);
+
+ provider = null;
+ }
+ }
+
+ return provider;
+ }
+
+ private AuditHandler getAuditProvider(Properties props, String propPrefix, AuditHandler consumer) {
+ AuditHandler ret;
+ String queueType = MiscUtil.getStringProperty(props, propPrefix + "." + "queuetype", DEFAULT_QUEUE_TYPE);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AuditProviderFactory.getAuditProvider() propPerfix={}, queueType={}", propPrefix, queueType);
+ }
+
+ if (FILE_QUEUE_TYPE.equalsIgnoreCase(queueType)) {
+ AuditFileQueue auditFileQueue = new AuditFileQueue(consumer);
+ String propPrefixFileQueue = propPrefix + "." + FILE_QUEUE_TYPE;
+
+ auditFileQueue.init(props, propPrefixFileQueue);
+
+ ret = new AuditBatchQueue(auditFileQueue);
+ } else {
+ ret = new AuditBatchQueue(consumer);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AuditProviderFactory.getAuditProvider()");
+ }
+
+ return ret;
+ }
+
+ private AuditHandler getDefaultProvider() {
+ return new DummyAuditProvider();
+ }
+
+ private void installJvmShutdownHook(Properties props) {
+ int shutdownHookMaxWaitSeconds = MiscUtil.getIntProperty(props, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT);
+
+ jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds);
+
+ String appType = this.componentAppType;
+
+ if (appType != null && !hbaseAppTypes.contains(appType)) {
+ ShutdownHookManager.get().addShutdownHook(jvmShutdownHook, RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY);
+ }
+ }
+
+ private static class RangerAsyncAuditCleanup implements Runnable {
+ final Semaphore startCleanup;
+ final Semaphore doneCleanup;
+ final AuditHandler mProvider;
+
+ RangerAsyncAuditCleanup(AuditHandler provider, Semaphore startCleanup, Semaphore doneCleanup) {
+ this.startCleanup = startCleanup;
+ this.doneCleanup = doneCleanup;
+ this.mProvider = provider;
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ LOG.info("RangerAsyncAuditCleanup: Waiting to audit cleanup start signal");
+
+ try {
+ startCleanup.acquire();
+ } catch (InterruptedException e) {
+ LOG.error("RangerAsyncAuditCleanup: Interrupted while waiting for audit startCleanup signal! Exiting the thread...", e);
+
+ break;
+ }
+
+ LOG.info("RangerAsyncAuditCleanup: Starting cleanup");
+
+ mProvider.waitToComplete();
+ mProvider.stop();
+ doneCleanup.release();
+
+ LOG.info("RangerAsyncAuditCleanup: Done cleanup");
+ }
+ }
+ }
+
+ private static class JVMShutdownHook extends Thread {
+ final Semaphore startCleanup = new Semaphore(0);
+ final Semaphore doneCleanup = new Semaphore(0);
+ final AtomicBoolean done = new AtomicBoolean(false);
+ final Thread cleanupThread;
+ final int maxWait;
+
+ public JVMShutdownHook(AuditHandler provider, int maxWait) {
+ this.maxWait = maxWait;
+
+ Runnable runnable = new RangerAsyncAuditCleanup(provider, startCleanup, doneCleanup);
+
+ cleanupThread = new Thread(runnable, "Ranger async Audit cleanup");
+
+ cleanupThread.setDaemon(true);
+ cleanupThread.start();
+ }
+
+ public void run() {
+ if (!done.compareAndSet(false, true)) {
+ LOG.info("==> JVMShutdownHook.run() already done by another thread");
+
+ return;
+ }
+
+ LOG.info("==> JVMShutdownHook.run()");
+ LOG.info("JVMShutdownHook: Signalling async audit cleanup to start.");
+
+ startCleanup.release();
+
+ try {
+ long start = System.currentTimeMillis();
+
+ LOG.info("JVMShutdownHook: Waiting up to {} seconds for audit cleanup to finish.", maxWait);
+
+ boolean cleanupFinishedInTime = doneCleanup.tryAcquire(maxWait, TimeUnit.SECONDS);
+
+ if (cleanupFinishedInTime) {
+ LOG.info("JVMShutdownHook: Audit cleanup finished after {} milli seconds", (System.currentTimeMillis() - start));
+ } else {
+ LOG.warn("JVMShutdownHook: could not detect finishing of audit cleanup even after waiting for {} seconds!", maxWait);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("JVMShutdownHook: Interrupted while waiting for completion of Async executor!", e);
+ }
+
+ LOG.info("JVMShutdownHook: Interrupting ranger async audit cleanup thread");
+
+ cleanupThread.interrupt();
+
+ LOG.info("<== JVMShutdownHook.run()");
+ }
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditWriterFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditWriterFactory.java
index 38844c95d..3e92dd145 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditWriterFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditWriterFactory.java
@@ -28,66 +28,73 @@ import java.util.Map;
import java.util.Properties;
public class AuditWriterFactory {
- private static final Logger logger = LoggerFactory.getLogger(AuditWriterFactory.class);
- public static final String AUDIT_FILETYPE_DEFAULT = "json";
- public static final String AUDIT_JSON_FILEWRITER_IMPL = "org.apache.ranger.audit.utils.RangerJSONAuditWriter";
- public static final String AUDIT_ORC_FILEWRITER_IMPL = "org.apache.ranger.audit.utils.RangerORCAuditWriter";
-
- public Map<String,String> auditConfigs = null;
- public Properties props = null;
- public String propPrefix = null;
- public String auditProviderName = null;
- public RangerAuditWriter auditWriter = null;
- private static volatile AuditWriterFactory me = null;
+ private static final Logger logger = LoggerFactory.getLogger(AuditWriterFactory.class);
+
+ public static final String AUDIT_FILETYPE_DEFAULT = "json";
+ public static final String AUDIT_JSON_FILEWRITER_IMPL = "org.apache.ranger.audit.utils.RangerJSONAuditWriter";
+ public static final String AUDIT_ORC_FILEWRITER_IMPL = "org.apache.ranger.audit.utils.RangerORCAuditWriter";
+
+ private static volatile AuditWriterFactory me;
+
+ public Map<String, String> auditConfigs;
+ public Properties props;
+ public String propPrefix;
+ public String auditProviderName;
+ public RangerAuditWriter auditWriter;
public static AuditWriterFactory getInstance() {
AuditWriterFactory auditWriter = me;
+
if (auditWriter == null) {
synchronized (AuditWriterFactory.class) {
auditWriter = me;
+
if (auditWriter == null) {
- me = auditWriter = new AuditWriterFactory();
+ auditWriter = new AuditWriterFactory();
+ me = auditWriter;
}
}
}
+
return auditWriter;
}
- public void init(Properties props, String propPrefix, String auditProviderName, Map<String,String> auditConfigs) throws Exception {
+ public void init(Properties props, String propPrefix, String auditProviderName, Map<String, String> auditConfigs) throws Exception {
if (logger.isDebugEnabled()) {
logger.debug("==> AuditWriterFactory.init()");
}
+
this.props = props;
this.propPrefix = propPrefix;
this.auditProviderName = auditProviderName;
this.auditConfigs = auditConfigs;
- String auditFileType = MiscUtil.getStringProperty(props, propPrefix + ".batch.filequeue.filetype", AUDIT_FILETYPE_DEFAULT);
- String writerClass = MiscUtil.getStringProperty(props, propPrefix + ".filewriter.impl");
+
+ String auditFileType = MiscUtil.getStringProperty(props, propPrefix + ".batch.filequeue.filetype", AUDIT_FILETYPE_DEFAULT);
+ String writerClass = MiscUtil.getStringProperty(props, propPrefix + ".filewriter.impl");
auditWriter = StringUtils.isEmpty(writerClass) ? createWriter(getDefaultWriter(auditFileType)) : createWriter(writerClass);
if (auditWriter != null) {
auditWriter.init(props, propPrefix, auditProviderName, auditConfigs);
+
if (logger.isDebugEnabled()) {
- logger.debug("<== AuditWriterFactory.init() :" + auditWriter.getClass().getName());
+ logger.debug("<== AuditWriterFactory.init() :{}", auditWriter.getClass().getName());
}
}
}
- public RangerAuditWriter createWriter(String writerClass) throws Exception {
+ public RangerAuditWriter createWriter(String writerClass) throws Exception {
if (logger.isDebugEnabled()) {
logger.debug("==> AuditWriterFactory.createWriter()");
}
- RangerAuditWriter ret = null;
- try {
- Class<RangerAuditWriter> cls = (Class<RangerAuditWriter>) Class.forName(writerClass);
- ret = cls.newInstance();
- } catch (Exception e) {
- throw e;
- }
+
+ Class<RangerAuditWriter> cls = (Class<RangerAuditWriter>) Class.forName(writerClass);
+ RangerAuditWriter ret = cls.newInstance();
+
if (logger.isDebugEnabled()) {
logger.debug("<== AuditWriterFactory.createWriter()");
}
+
return ret;
}
@@ -95,7 +102,9 @@ public class AuditWriterFactory {
if (logger.isDebugEnabled()) {
logger.debug("==> AuditWriterFactory.getDefaultWriter()");
}
- String ret = null;
+
+ final String ret;
+
switch (auditFileType) {
case "orc":
ret = AUDIT_ORC_FILEWRITER_IMPL;
@@ -103,14 +112,19 @@ public class AuditWriterFactory {
case "json":
ret = AUDIT_JSON_FILEWRITER_IMPL;
break;
+ default:
+ ret = null;
+ break;
}
+
if (logger.isDebugEnabled()) {
- logger.debug("<== AuditWriterFactory.getDefaultWriter() :" + ret);
+ logger.debug("<== AuditWriterFactory.getDefaultWriter() :{}", ret);
}
+
return ret;
}
- public RangerAuditWriter getAuditWriter(){
+ public RangerAuditWriter getAuditWriter() {
return this.auditWriter;
}
-}
\ No newline at end of file
+}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index 94c6d754b..a56ab12bb 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -23,472 +23,481 @@ import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicLong;
-
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicLong;
+
public abstract class BaseAuditHandler implements AuditHandler {
- private static final Logger LOG = LoggerFactory.getLogger(BaseAuditHandler.class);
-
- static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms";
-
- static final String AUDIT_LOG_STATUS_LOG_ENABLED = "xasecure.audit.log.status.log.enabled";
- static final String AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = "xasecure.audit.log.status.log.interval.sec";
- static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED = false;
- static final long DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5L * 60; // 5 minutes
-
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore";
- public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks";
-
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore";
- public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks";
-
- public static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm();
- public static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm();
- public static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2";
-
- public static final String PROP_CONFIG = "config";
- public static final String FAILED_TO_LOG_AUDIT_EVENT = "failed to log audit event: {}";
- private int mLogFailureReportMinIntervalInMs = 60 * 1000;
-
- private AtomicLong mFailedLogLastReportTime = new AtomicLong(0);
- private AtomicLong mFailedLogCountSinceLastReport = new AtomicLong(0);
- private AtomicLong mFailedLogCountLifeTime = new AtomicLong(0);
-
- public static final String PROP_NAME = "name";
- public static final String PROP_CLASS_NAME = "classname";
-
- public static final String PROP_DEFAULT_PREFIX = "xasecure.audit.provider";
-
- protected String propPrefix = PROP_DEFAULT_PREFIX;
-
- protected String providerName = null;
- protected String parentPath = null;
-
- protected int failedRetryTimes = 3;
- protected int failedRetrySleep = 3 * 1000;
-
- int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
- long lastErrorLogMS = 0;
-
- long totalCount = 0;
- long totalSuccessCount = 0;
- long totalFailedCount = 0;
- long totalStashedCount = 0;
- long totalDeferredCount = 0;
-
- long lastIntervalCount = 0;
- long lastIntervalSuccessCount = 0;
- long lastIntervalFailedCount = 0;
- long lastStashedCount = 0;
- long lastDeferredCount = 0;
-
- boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
- long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
- long lastStatusLogTime = System.currentTimeMillis();
- long nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
-
- protected Properties props = null;
- protected Map<String, String> configProps = new HashMap<>();
-
- @Override
- public void init(Properties props) {
- init(props, null);
- }
-
- @Override
- public void init(Properties props, String basePropertyName) {
- LOG.info("BaseAuditProvider.init()");
- this.props = props;
- if (basePropertyName != null) {
- propPrefix = basePropertyName;
- }
- LOG.info("propPrefix=" + propPrefix);
-
- String name = MiscUtil.getStringProperty(props, basePropertyName + "."
- + PROP_NAME);
- if (name != null && !name.isEmpty()) {
- setName(name);
- }
- // Get final token
- if (providerName == null) {
- List<String> tokens = MiscUtil.toArray(propPrefix, ".");
- if (!tokens.isEmpty()) {
- String finalToken = tokens.get(tokens.size() - 1);
- setName(finalToken);
- LOG.info("Using providerName from property prefix. providerName="
- + getName());
- }
- }
- LOG.info("providerName=" + getName());
-
- mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props,
- AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
-
- boolean globalStatusLogEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED, DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED);
- long globalStatusLogIntervalSec = MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC);
-
- statusLogEnabled = MiscUtil.getBooleanProperty(props, basePropertyName + ".status.log.enabled", globalStatusLogEnabled);
- statusLogIntervalMS = MiscUtil.getLongProperty(props, basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) * 1000;
-
- nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
-
- LOG.info(AUDIT_LOG_STATUS_LOG_ENABLED + "=" + globalStatusLogEnabled);
- LOG.info(AUDIT_LOG_STATUS_LOG_INTERVAL_SEC + "=" + globalStatusLogIntervalSec);
- LOG.info(basePropertyName + ".status.log.enabled=" + statusLogEnabled);
- LOG.info(basePropertyName + ".status.log.interval.sec=" + (statusLogIntervalMS / 1000));
-
- String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + ".";
- for (Object propNameObj : props.keySet()) {
- String propName = propNameObj.toString();
-
- if (!propName.startsWith(configPropsNamePrefix)) {
- continue;
- }
- String configName = propName.substring(configPropsNamePrefix.length());
- String configValue = props.getProperty(propName);
- configProps.put(configName, configValue);
- LOG.info("Found Config property: " + configName + " => " + configValue);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
- * audit.model.AuditEventBase)
- */
- @Override
- public boolean log(AuditEventBase event) {
- return log(Collections.singletonList(event));
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.lang.String)
- */
- @Override
- public boolean logJSON(String event) {
- AuditEventBase eventObj = MiscUtil.fromJson(event,
- AuthzAuditEvent.class);
- return log(eventObj);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.util.Collection
- * )
- */
- @Override
- public boolean logJSON(Collection<String> events) {
- List<AuditEventBase> eventList = new ArrayList<>(events.size());
- for (String event : events) {
- eventList.add(MiscUtil.fromJson(event, AuthzAuditEvent.class));
- }
- return log(eventList);
- }
-
- @Override
- public boolean logFile(File file) {
- return logFile(file);
- }
-
- public String getParentPath() {
- return parentPath;
- }
-
- public void setParentPath(String parentPath) {
- this.parentPath = parentPath;
- }
-
- public String getFinalPath() {
- return getName();
- }
-
- public void setName(String name) {
- providerName = name;
- }
-
- @Override
- public String getName() {
- if (parentPath != null) {
- return parentPath + "." + providerName;
- }
- return providerName;
- }
-
- public long addTotalCount(int count) {
- totalCount += count;
- return totalCount;
- }
-
- public long addSuccessCount(int count) {
- totalSuccessCount += count;
- return totalSuccessCount;
- }
-
- public long addFailedCount(int count) {
- totalFailedCount += count;
- return totalFailedCount;
- }
-
- public long addStashedCount(int count) {
- totalStashedCount += count;
- return totalStashedCount;
- }
-
- public long addDeferredCount(int count) {
- totalDeferredCount += count;
- return totalDeferredCount;
- }
-
- public long getTotalCount() {
- return totalCount;
- }
-
- public long getTotalSuccessCount() {
- return totalSuccessCount;
- }
-
- public long getTotalFailedCount() {
- return totalFailedCount;
- }
-
- public long getTotalStashedCount() {
- return totalStashedCount;
- }
-
- public long getLastStashedCount() {
- return lastStashedCount;
- }
-
- public long getTotalDeferredCount() {
- return totalDeferredCount;
- }
-
- public long getLastDeferredCount() {
- return lastDeferredCount;
- }
-
- public boolean isStatusLogEnabled() { return statusLogEnabled; }
-
- public void logStatusIfRequired() {
- if (System.currentTimeMillis() > nextStatusLogTime) {
- logStatus();
- }
- }
-
- public void logStatus() {
- try {
- long currTime = System.currentTimeMillis();
- long diffTime = currTime - lastStatusLogTime;
-
- lastStatusLogTime = currTime;
- nextStatusLogTime = currTime + statusLogIntervalMS;
-
- long diffCount = totalCount - lastIntervalCount;
- long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
- long diffFailed = totalFailedCount - lastIntervalFailedCount;
- long diffStashed = totalStashedCount - lastStashedCount;
- long diffDeferred = totalDeferredCount - lastDeferredCount;
-
- if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0
- && diffStashed == 0 && diffDeferred == 0) {
- return;
- }
-
- lastIntervalCount = totalCount;
- lastIntervalSuccessCount = totalSuccessCount;
- lastIntervalFailedCount = totalFailedCount;
- lastStashedCount = totalStashedCount;
- lastDeferredCount = totalDeferredCount;
-
- if (statusLogEnabled) {
- String finalPath = "";
- String tFinalPath = getFinalPath();
- if (!getName().equals(tFinalPath)) {
- finalPath = ", finalDestination=" + tFinalPath;
- }
-
- logAuditStatus(diffTime, diffCount, diffSuccess, diffFailed, diffStashed, diffDeferred, finalPath);
- }
- } catch (Exception t) {
- LOG.error("Error while printing stats. auditProvider=" + getName());
- }
- }
- private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) {
- String msg = "Audit Status Log: name="
- + getName()
- + finalPath
- + ", interval="
- + formatIntervalForLog(diffTime)
- + ", events="
- + diffCount
- + (diffSuccess > 0 ? (", succcessCount=" + diffSuccess)
- : "")
- + (diffFailed > 0 ? (", failedCount=" + diffFailed) : "")
- + (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "")
- + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
- : "")
- + ", totalEvents="
- + totalCount
- + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount)
- : "")
- + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount)
- : "")
- + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount)
- : "")
- + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
- : "");
- LOG.info(msg);
- }
-
- public void logError(String msg, Object arg) {
- long currTimeMS = System.currentTimeMillis();
- if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
- LOG.error(msg, arg);
- lastErrorLogMS = currTimeMS;
- }
- }
-
- public void logError(String msg, Throwable ex) {
- long currTimeMS = System.currentTimeMillis();
- if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
- LOG.error(msg, ex);
- lastErrorLogMS = currTimeMS;
- }
- }
-
- public String getTimeDiffStr(long time1, long time2) {
- long timeInMs = Math.abs(time1 - time2);
- return formatIntervalForLog(timeInMs);
- }
-
- public String formatIntervalForLog(long timeInMs) {
- long hours = timeInMs / (60 * 60 * 1000);
- long minutes = (timeInMs / (60 * 1000)) % 60;
- long seconds = (timeInMs % (60 * 1000)) / 1000;
- long mSeconds = (timeInMs % (1000));
-
- if (hours > 0)
- return String.format("%02d:%02d:%02d.%03d hours", hours, minutes,
- seconds, mSeconds);
- else if (minutes > 0)
- return String.format("%02d:%02d.%03d minutes", minutes, seconds,
- mSeconds);
- else if (seconds > 0)
- return String.format("%02d.%03d seconds", seconds, mSeconds);
- else
- return String.format("%03d milli-seconds", mSeconds);
- }
-
- public void logFailedEvent(AuditEventBase event) {
- logFailedEvent(event, "");
- }
-
- public void logFailedEvent(AuditEventBase event, Throwable excp) {
- long now = System.currentTimeMillis();
-
- long timeSinceLastReport = now - mFailedLogLastReportTime.get();
- long countSinceLastReport = mFailedLogCountSinceLastReport
- .incrementAndGet();
- long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
-
- if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
- mFailedLogLastReportTime.set(now);
- mFailedLogCountSinceLastReport.set(0);
-
- if (excp != null) {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event), excp);
- } else {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event));
- }
-
- if (countLifeTime > 1) { // no stats to print for the 1st failure
- LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
- }
- }
- }
-
- public void logFailedEvent(Collection<AuditEventBase> events) {
- logFailedEvent(events, "");
- }
-
- public void logFailedEvent(Collection<AuditEventBase> events, Throwable excp) {
- for (AuditEventBase event : events) {
- logFailedEvent(event, excp);
- }
- }
-
- public void logFailedEvent(AuditEventBase event, String message) {
- long now = System.currentTimeMillis();
-
- long timeSinceLastReport = now - mFailedLogLastReportTime.get();
- long countSinceLastReport = mFailedLogCountSinceLastReport
- .incrementAndGet();
- long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
-
- if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
- mFailedLogLastReportTime.set(now);
- mFailedLogCountSinceLastReport.set(0);
-
- LOG.warn("failed to log audit event: {} , errorMessage={}", MiscUtil.stringify(event), message);
-
- if (countLifeTime > 1) { // no stats to print for the 1st failure
- LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
- }
- }
- }
-
- public void logFailedEvent(Collection<AuditEventBase> events,
- String errorMessage) {
- for (AuditEventBase event : events) {
- logFailedEvent(event, errorMessage);
- }
- }
-
- public void logFailedEventJSON(String event, Throwable excp) {
- long now = System.currentTimeMillis();
-
- long timeSinceLastReport = now - mFailedLogLastReportTime.get();
- long countSinceLastReport = mFailedLogCountSinceLastReport
- .incrementAndGet();
- long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
-
- if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
- mFailedLogLastReportTime.set(now);
- mFailedLogCountSinceLastReport.set(0);
-
- if (excp != null) {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event, excp);
- } else {
- LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event);
- }
-
- if (countLifeTime > 1) { // no stats to print for the 1st failure
- LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
- }
- }
- }
-
- public void logFailedEventJSON(Collection<String> events, Throwable excp) {
- for (String event : events) {
- logFailedEventJSON(event, excp);
- }
- }
+ private static final Logger LOG = LoggerFactory.getLogger(BaseAuditHandler.class);
+
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL_ALIAS = "sslKeyStore";
+ public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE_DEFAULT = "jks";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE = "xasecure.policymgr.clientssl.truststore";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE = "xasecure.policymgr.clientssl.truststore.type";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.truststore.credential.file";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_CREDENTIAL_ALIAS = "sslTrustStore";
+ public static final String RANGER_POLICYMGR_TRUSTSTORE_FILE_TYPE_DEFAULT = "jks";
+ public static final String RANGER_SSL_KEYMANAGER_ALGO_TYPE = KeyManagerFactory.getDefaultAlgorithm();
+ public static final String RANGER_SSL_TRUSTMANAGER_ALGO_TYPE = TrustManagerFactory.getDefaultAlgorithm();
+ public static final String RANGER_SSL_CONTEXT_ALGO_TYPE = "TLSv1.2";
+ public static final String PROP_CONFIG = "config";
+ public static final String FAILED_TO_LOG_AUDIT_EVENT = "failed to log audit event: {}";
+ public static final String PROP_NAME = "name";
+ public static final String PROP_CLASS_NAME = "classname";
+ public static final String PROP_DEFAULT_PREFIX = "xasecure.audit.provider";
+
+ static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms";
+ static final String AUDIT_LOG_STATUS_LOG_ENABLED = "xasecure.audit.log.status.log.enabled";
+ static final String AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = "xasecure.audit.log.status.log.interval.sec";
+ static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED = false;
+ static final long DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5L * 60; // 5 minutes
+
+ protected String propPrefix = PROP_DEFAULT_PREFIX;
+ protected String providerName;
+ protected String parentPath;
+ protected int failedRetryTimes = 3;
+ protected int failedRetrySleep = 3 * 1000;
+ protected Map<String, String> configProps = new HashMap<>();
+ protected Properties props;
+
+ int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
+ long lastErrorLogMS;
+ long totalCount;
+ long totalSuccessCount;
+ long totalFailedCount;
+ long totalStashedCount;
+ long totalDeferredCount;
+ long lastIntervalCount;
+ long lastIntervalSuccessCount;
+ long lastIntervalFailedCount;
+ long lastStashedCount;
+ long lastDeferredCount;
+ boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
+ long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
+ long lastStatusLogTime = System.currentTimeMillis();
+ long nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
+
+ private int mLogFailureReportMinIntervalInMs = 60 * 1000;
+ private final AtomicLong mFailedLogLastReportTime = new AtomicLong(0);
+ private final AtomicLong mFailedLogCountSinceLastReport = new AtomicLong(0);
+ private final AtomicLong mFailedLogCountLifeTime = new AtomicLong(0);
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
+ * audit.model.AuditEventBase)
+ */
+ @Override
+ public boolean log(AuditEventBase event) {
+ return log(Collections.singletonList(event));
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.lang.String)
+ */
+ @Override
+ public boolean logJSON(String event) {
+ AuditEventBase eventObj = MiscUtil.fromJson(event, AuthzAuditEvent.class);
+
+ return log(eventObj);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#logJSON(java.util.Collection)
+ */
+ @Override
+ public boolean logJSON(Collection<String> events) {
+ List<AuditEventBase> eventList = new ArrayList<>(events.size());
+
+ for (String event : events) {
+ eventList.add(MiscUtil.fromJson(event, AuthzAuditEvent.class));
+ }
+
+ return log(eventList);
+ }
+
+ @Override
+ public boolean logFile(File file) {
+ return false;
+ }
+
+ @Override
+ public void init(Properties props) {
+ init(props, null);
+ }
+
+ @Override
+ public void init(Properties props, String basePropertyName) {
+ LOG.info("BaseAuditProvider.init()");
+
+ this.props = props;
+
+ if (basePropertyName != null) {
+ propPrefix = basePropertyName;
+ }
+
+ LOG.info("propPrefix={}", propPrefix);
+
+ String name = MiscUtil.getStringProperty(props, basePropertyName + "." + PROP_NAME);
+
+ if (name != null && !name.isEmpty()) {
+ setName(name);
+ }
+
+ // Get final token
+ if (providerName == null) {
+ List<String> tokens = MiscUtil.toArray(propPrefix, ".");
+
+ if (!tokens.isEmpty()) {
+ String finalToken = tokens.get(tokens.size() - 1);
+
+ setName(finalToken);
+
+ LOG.info("Using providerName from property prefix. providerName={}", getName());
+ }
+ }
+
+ LOG.info("providerName={}", getName());
+
+ mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
+
+ boolean globalStatusLogEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED, DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED);
+ long globalStatusLogIntervalSec = MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC);
+
+ statusLogEnabled = MiscUtil.getBooleanProperty(props, basePropertyName + ".status.log.enabled", globalStatusLogEnabled);
+ statusLogIntervalMS = MiscUtil.getLongProperty(props, basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) * 1000;
+ nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
+
+ LOG.info("{}={}", AUDIT_LOG_STATUS_LOG_ENABLED, globalStatusLogEnabled);
+ LOG.info("{}={}", AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, globalStatusLogIntervalSec);
+ LOG.info("{}.status.log.enabled={}", basePropertyName, statusLogEnabled);
+ LOG.info("{}.status.log.interval.sec={}", basePropertyName, (statusLogIntervalMS / 1000));
+
+ String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + ".";
+
+ for (Object propNameObj : props.keySet()) {
+ String propName = propNameObj.toString();
+
+ if (!propName.startsWith(configPropsNamePrefix)) {
+ continue;
+ }
+
+ String configName = propName.substring(configPropsNamePrefix.length());
+ String configValue = props.getProperty(propName);
+
+ configProps.put(configName, configValue);
+
+ LOG.info("Found Config property: {} => {}", configName, configValue);
+ }
+ }
+
+ @Override
+ public String getName() {
+ if (parentPath != null) {
+ return parentPath + "." + providerName;
+ }
+
+ return providerName;
+ }
+
+ public void setName(String name) {
+ providerName = name;
+ }
+
+ public String getParentPath() {
+ return parentPath;
+ }
+
+ public void setParentPath(String parentPath) {
+ this.parentPath = parentPath;
+ }
+
+ public String getFinalPath() {
+ return getName();
+ }
+
+ public long addTotalCount(int count) {
+ totalCount += count;
+
+ return totalCount;
+ }
+
+ public long addSuccessCount(int count) {
+ totalSuccessCount += count;
+
+ return totalSuccessCount;
+ }
+
+ public long addFailedCount(int count) {
+ totalFailedCount += count;
+
+ return totalFailedCount;
+ }
+
+ public long addStashedCount(int count) {
+ totalStashedCount += count;
+
+ return totalStashedCount;
+ }
+
+ public long addDeferredCount(int count) {
+ totalDeferredCount += count;
+
+ return totalDeferredCount;
+ }
+
+ public long getTotalCount() {
+ return totalCount;
+ }
+
+ public long getTotalSuccessCount() {
+ return totalSuccessCount;
+ }
+
+ public long getTotalFailedCount() {
+ return totalFailedCount;
+ }
+
+ public long getTotalStashedCount() {
+ return totalStashedCount;
+ }
+
+ public long getLastStashedCount() {
+ return lastStashedCount;
+ }
+
+ public long getTotalDeferredCount() {
+ return totalDeferredCount;
+ }
+
+ public long getLastDeferredCount() {
+ return lastDeferredCount;
+ }
+
+ public boolean isStatusLogEnabled() {
+ return statusLogEnabled;
+ }
+ public void logStatusIfRequired() {
+ if (System.currentTimeMillis() > nextStatusLogTime) {
+ logStatus();
+ }
+ }
+
+ public void logStatus() {
+ try {
+ long currTime = System.currentTimeMillis();
+ long diffTime = currTime - lastStatusLogTime;
+
+ lastStatusLogTime = currTime;
+ nextStatusLogTime = currTime + statusLogIntervalMS;
+
+ long diffCount = totalCount - lastIntervalCount;
+ long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
+ long diffFailed = totalFailedCount - lastIntervalFailedCount;
+ long diffStashed = totalStashedCount - lastStashedCount;
+ long diffDeferred = totalDeferredCount - lastDeferredCount;
+
+ if (diffCount == 0 && diffSuccess == 0 && diffFailed == 0 && diffStashed == 0 && diffDeferred == 0) {
+ return;
+ }
+
+ lastIntervalCount = totalCount;
+ lastIntervalSuccessCount = totalSuccessCount;
+ lastIntervalFailedCount = totalFailedCount;
+ lastStashedCount = totalStashedCount;
+ lastDeferredCount = totalDeferredCount;
+
+ if (statusLogEnabled) {
+ String finalPath = "";
+ String tFinalPath = getFinalPath();
+
+ if (!getName().equals(tFinalPath)) {
+ finalPath = ", finalDestination=" + tFinalPath;
+ }
+
+ logAuditStatus(diffTime, diffCount, diffSuccess, diffFailed, diffStashed, diffDeferred, finalPath);
+ }
+ } catch (Exception t) {
+ LOG.error("Error while printing stats. auditProvider={}", getName());
+ }
+ }
+
+ public void logError(String msg, Object arg) {
+ long currTimeMS = System.currentTimeMillis();
+
+ if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
+ LOG.error(msg, arg);
+
+ lastErrorLogMS = currTimeMS;
+ }
+ }
+
+ public void logError(String msg, Throwable ex) {
+ long currTimeMS = System.currentTimeMillis();
+
+ if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {
+ LOG.error(msg, ex);
+
+ lastErrorLogMS = currTimeMS;
+ }
+ }
+
+ public String getTimeDiffStr(long time1, long time2) {
+ long timeInMs = Math.abs(time1 - time2);
+
+ return formatIntervalForLog(timeInMs);
+ }
+
+ public String formatIntervalForLog(long timeInMs) {
+ long hours = timeInMs / (60 * 60 * 1000);
+ long minutes = (timeInMs / (60 * 1000)) % 60;
+ long seconds = (timeInMs % (60 * 1000)) / 1000;
+ long mSeconds = (timeInMs % (1000));
+
+ if (hours > 0) {
+ return String.format("%02d:%02d:%02d.%03d hours", hours, minutes, seconds, mSeconds);
+ } else if (minutes > 0) {
+ return String.format("%02d:%02d.%03d minutes", minutes, seconds, mSeconds);
+ } else if (seconds > 0) {
+ return String.format("%02d.%03d seconds", seconds, mSeconds);
+ } else {
+ return String.format("%03d milli-seconds", mSeconds);
+ }
+ }
+
+ public void logFailedEvent(AuditEventBase event) {
+ logFailedEvent(event, "");
+ }
+
+ public void logFailedEvent(AuditEventBase event, Throwable excp) {
+ long now = System.currentTimeMillis();
+ long timeSinceLastReport = now - mFailedLogLastReportTime.get();
+ long countSinceLastReport = mFailedLogCountSinceLastReport.incrementAndGet();
+ long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
+
+ if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
+ mFailedLogLastReportTime.set(now);
+ mFailedLogCountSinceLastReport.set(0);
+
+ if (excp != null) {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event), excp);
+ } else {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, MiscUtil.stringify(event));
+ }
+
+ if (countLifeTime > 1) { // no stats to print for the 1st failure
+ LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
+ }
+ }
+ }
+
+ public void logFailedEvent(Collection<AuditEventBase> events) {
+ logFailedEvent(events, "");
+ }
+
+ public void logFailedEvent(Collection<AuditEventBase> events, Throwable excp) {
+ for (AuditEventBase event : events) {
+ logFailedEvent(event, excp);
+ }
+ }
+
+ public void logFailedEvent(AuditEventBase event, String message) {
+ long now = System.currentTimeMillis();
+ long timeSinceLastReport = now - mFailedLogLastReportTime.get();
+ long countSinceLastReport = mFailedLogCountSinceLastReport.incrementAndGet();
+ long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
+
+ if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
+ mFailedLogLastReportTime.set(now);
+ mFailedLogCountSinceLastReport.set(0);
+
+ LOG.warn("failed to log audit event: {} , errorMessage={}", MiscUtil.stringify(event), message);
+
+ if (countLifeTime > 1) { // no stats to print for the 1st failure
+ LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
+ }
+ }
+ }
+
+ public void logFailedEvent(Collection<AuditEventBase> events, String errorMessage) {
+ for (AuditEventBase event : events) {
+ logFailedEvent(event, errorMessage);
+ }
+ }
+
+ public void logFailedEventJSON(String event, Throwable excp) {
+ long now = System.currentTimeMillis();
+ long timeSinceLastReport = now - mFailedLogLastReportTime.get();
+ long countSinceLastReport = mFailedLogCountSinceLastReport.incrementAndGet();
+ long countLifeTime = mFailedLogCountLifeTime.incrementAndGet();
+
+ if (timeSinceLastReport >= mLogFailureReportMinIntervalInMs) {
+ mFailedLogLastReportTime.set(now);
+ mFailedLogCountSinceLastReport.set(0);
+
+ if (excp != null) {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event, excp);
+ } else {
+ LOG.warn(FAILED_TO_LOG_AUDIT_EVENT, event);
+ }
+
+ if (countLifeTime > 1) { // no stats to print for the 1st failure
+ LOG.warn("Log failure count: {} in past {}; {} during process lifetime", countSinceLastReport, formatIntervalForLog(timeSinceLastReport), countLifeTime);
+ }
+ }
+ }
+
+ public void logFailedEventJSON(Collection<String> events, Throwable excp) {
+ for (String event : events) {
+ logFailedEventJSON(event, excp);
+ }
+ }
+
+ private void logAuditStatus(long diffTime, long diffCount, long diffSuccess, long diffFailed, long diffStashed, long diffDeferred, String finalPath) {
+ String msg = "Audit Status Log: name="
+ + getName()
+ + finalPath
+ + ", interval="
+ + formatIntervalForLog(diffTime)
+ + ", events="
+ + diffCount
+ + (diffSuccess > 0 ? (", succcessCount=" + diffSuccess)
+ : "")
+ + (diffFailed > 0 ? (", failedCount=" + diffFailed) : "")
+ + (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "")
+ + (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
+ : "")
+ + ", totalEvents="
+ + totalCount
+ + (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount)
+ : "")
+ + (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount)
+ : "")
+ + (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount)
+ : "")
+ + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
+ : "");
+ LOG.info(msg);
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BufferedAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BufferedAuditProvider.java
index c80d045b0..c5dccd2a5 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BufferedAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BufferedAuditProvider.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,104 +17,104 @@
*/
package org.apache.ranger.audit.provider;
-import java.util.Collection;
-
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.model.AuthzAuditEvent;
+import java.util.Collection;
+
public abstract class BufferedAuditProvider extends BaseAuditHandler {
- private LogBuffer<AuditEventBase> mBuffer = null;
- private LogDestination<AuditEventBase> mDestination = null;
-
- @Override
- public boolean log(AuditEventBase event) {
- if (event instanceof AuthzAuditEvent) {
- AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
-
- if (authzEvent.getAgentHostname() == null) {
- authzEvent.setAgentHostname(MiscUtil.getHostname());
- }
-
- if (authzEvent.getLogType() == null) {
- authzEvent.setLogType("RangerAudit");
- }
-
- if (authzEvent.getEventId() == null) {
- authzEvent.setEventId(MiscUtil.generateUniqueId());
- }
- }
-
- if (!mBuffer.add(event)) {
- logFailedEvent(event);
- return false;
- }
- return true;
- }
-
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- boolean ret = true;
- for (AuditEventBase event : events) {
- ret = log(event);
- if (!ret) {
- break;
- }
- }
- return ret;
- }
-
- @Override
- public boolean logJSON(String event) {
- AuditEventBase eventObj = MiscUtil.fromJson(event,
- AuthzAuditEvent.class);
- return log(eventObj);
- }
-
- @Override
- public boolean logJSON(Collection<String> events) {
- boolean ret = true;
- for (String event : events) {
- ret = logJSON(event);
- if (!ret) {
- break;
- }
- }
- return ret;
- }
-
- @Override
- public void start() {
- mBuffer.start(mDestination);
- }
-
- @Override
- public void stop() {
- mBuffer.stop();
- }
-
- @Override
- public void waitToComplete() {
- }
-
- @Override
- public void waitToComplete(long timeout) {
- }
-
- @Override
- public void flush() {
- }
-
- protected LogBuffer<AuditEventBase> getBuffer() {
- return mBuffer;
- }
-
- protected LogDestination<AuditEventBase> getDestination() {
- return mDestination;
- }
-
- protected void setBufferAndDestination(LogBuffer<AuditEventBase> buffer,
- LogDestination<AuditEventBase> destination) {
- mBuffer = buffer;
- mDestination = destination;
- }
+ private LogBuffer<AuditEventBase> mBuffer;
+ private LogDestination<AuditEventBase> mDestination;
+
+ @Override
+ public boolean log(AuditEventBase event) {
+ if (event instanceof AuthzAuditEvent) {
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+
+ if (authzEvent.getAgentHostname() == null) {
+ authzEvent.setAgentHostname(MiscUtil.getHostname());
+ }
+
+ if (authzEvent.getLogType() == null) {
+ authzEvent.setLogType("RangerAudit");
+ }
+
+ if (authzEvent.getEventId() == null) {
+ authzEvent.setEventId(MiscUtil.generateUniqueId());
+ }
+ }
+
+ if (!mBuffer.add(event)) {
+ logFailedEvent(event);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean logJSON(String event) {
+ AuditEventBase eventObj = MiscUtil.fromJson(event,
+ AuthzAuditEvent.class);
+ return log(eventObj);
+ }
+
+ @Override
+ public boolean logJSON(Collection<String> events) {
+ boolean ret = true;
+ for (String event : events) {
+ ret = logJSON(event);
+ if (!ret) {
+ break;
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ boolean ret = true;
+ for (AuditEventBase event : events) {
+ ret = log(event);
+ if (!ret) {
+ break;
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public void start() {
+ mBuffer.start(mDestination);
+ }
+
+ @Override
+ public void stop() {
+ mBuffer.stop();
+ }
+
+ @Override
+ public void waitToComplete() {
+ }
+
+ @Override
+ public void waitToComplete(long timeout) {
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ protected LogBuffer<AuditEventBase> getBuffer() {
+ return mBuffer;
+ }
+
+ protected LogDestination<AuditEventBase> getDestination() {
+ return mDestination;
+ }
+
+ protected void setBufferAndDestination(LogBuffer<AuditEventBase> buffer,
+ LogDestination<AuditEventBase> destination) {
+ mBuffer = buffer;
+ mDestination = destination;
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DebugTracer.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DebugTracer.java
index 7396fd048..dbf591403 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DebugTracer.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DebugTracer.java
@@ -17,12 +17,19 @@
package org.apache.ranger.audit.provider;
public interface DebugTracer {
- void debug(String msg);
- void debug(String msg, Throwable excp);
- void info(String msg);
- void info(String msg, Throwable excp);
- void warn(String msg);
- void warn(String msg, Throwable excp);
- void error(String msg);
- void error(String msg, Throwable excp);
+ void debug(String msg);
+
+ void debug(String msg, Throwable excp);
+
+ void info(String msg);
+
+ void info(String msg, Throwable excp);
+
+ void warn(String msg);
+
+ void warn(String msg, Throwable excp);
+
+ void error(String msg);
+
+ void error(String msg, Throwable excp);
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java
index cbd25ab7c..f6174f650 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DummyAuditProvider.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,99 +17,97 @@
*/
package org.apache.ranger.audit.provider;
-import java.io.File;
-import java.util.Collection;
-import java.util.Properties;
-
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.model.AuthzAuditEvent;
+import java.io.File;
+import java.util.Collection;
+import java.util.Properties;
public class DummyAuditProvider implements AuditHandler {
- @Override
- public void init(Properties prop) {
- // intentionally left empty
- }
-
- @Override
- public boolean log(AuditEventBase event) {
- // intentionally left empty
- return true;
- }
-
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- for (AuditEventBase event : events) {
- log(event);
- }
- return true;
- }
-
- @Override
- public boolean logJSON(String event) {
- AuditEventBase eventObj = MiscUtil.fromJson(event,
- AuthzAuditEvent.class);
- return log(eventObj);
- }
-
- @Override
- public boolean logJSON(Collection<String> events) {
- for (String event : events) {
- logJSON(event);
- }
- return false;
- }
-
- @Override
- public void start() {
- // intentionally left empty
- }
-
- @Override
- public void stop() {
- // intentionally left empty
- }
-
- @Override
- public void waitToComplete() {
- // intentionally left empty
- }
-
- @Override
- public void flush() {
- // intentionally left empty
- }
-
- /* (non-Javadoc)
- * @see org.apache.ranger.audit.provider.AuditProvider#init(java.util.Properties, java.lang.String)
- */
- @Override
- public void init(Properties prop, String basePropertyName) {
- // intentionally left empty
- }
-
- /* (non-Javadoc)
- * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete(long)
- */
- @Override
- public void waitToComplete(long timeout) {
- // intentionally left empty
- }
-
- /* (non-Javadoc)
- * @see org.apache.ranger.audit.provider.AuditProvider#getName()
- */
- @Override
- public String getName() {
- return this.getClass().getName();
- }
-
- /* (non-Javadoc)
- * @see org.apache.ranger.audit.provider.AuditProvider#getAuditFileType()
- */
- @Override
- public boolean logFile(File file) {
- return logFile(file);
- }
-
+ @Override
+ public boolean log(AuditEventBase event) {
+ // intentionally left empty
+ return true;
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ for (AuditEventBase event : events) {
+ log(event);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean logJSON(String event) {
+ AuditEventBase eventObj = MiscUtil.fromJson(event,
+ AuthzAuditEvent.class);
+ return log(eventObj);
+ }
+
+ @Override
+ public boolean logJSON(Collection<String> events) {
+ for (String event : events) {
+ logJSON(event);
+ }
+ return false;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.ranger.audit.provider.AuditProvider#getAuditFileType()
+ */
+ @Override
+ public boolean logFile(File file) {
+ return logFile(file);
+ }
+
+ @Override
+ public void init(Properties prop) {
+ // intentionally left empty
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.ranger.audit.provider.AuditProvider#init(java.util.Properties, java.lang.String)
+ */
+ @Override
+ public void init(Properties prop, String basePropertyName) {
+ // intentionally left empty
+ }
+
+ @Override
+ public void start() {
+ // intentionally left empty
+ }
+
+ @Override
+ public void stop() {
+ // intentionally left empty
+ }
+
+ @Override
+ public void waitToComplete() {
+ // intentionally left empty
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete(long)
+ */
+ @Override
+ public void waitToComplete(long timeout) {
+ // intentionally left empty
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.ranger.audit.provider.AuditProvider#getName()
+ */
+ @Override
+ public String getName() {
+ return this.getClass().getName();
+ }
+
+ @Override
+ public void flush() {
+ // intentionally left empty
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
index d720ebccc..e08c5b808 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java
@@ -18,10 +18,11 @@
*/
package org.apache.ranger.audit.provider;
+import org.apache.hadoop.security.UserGroupInformation;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -34,658 +35,642 @@ import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.TreeSet;
-import org.apache.hadoop.security.UserGroupInformation;
+public class LocalFileLogBuffer<T> implements LogBuffer<T> {
+ private String mDirectory;
+ private String mFile;
+ private int mFlushIntervalSeconds = 1 * 60;
+ private int mFileBufferSizeBytes = 8 * 1024;
+ private String mEncoding;
+ private boolean mIsAppend = true;
+ private int mRolloverIntervalSeconds = 10 * 60;
+ private String mArchiveDirectory;
+ private int mArchiveFileCount = 10;
+ private final DebugTracer mLogger;
+ private Writer mWriter;
+ private String mBufferFilename;
+ private long mNextRolloverTime;
+ private long mNextFlushTime;
+ private final int mFileOpenRetryIntervalInMs = 60 * 1000;
+ private long mNextFileOpenRetryTime;
+ private DestinationDispatcherThread<T> mDispatcherThread;
-public class LocalFileLogBuffer<T> implements LogBuffer<T> {
- private String mDirectory = null;
- private String mFile = null;
- private int mFlushIntervalSeconds = 1 * 60;
- private int mFileBufferSizeBytes = 8 * 1024;
- private String mEncoding = null;
- private boolean mIsAppend = true;
- private int mRolloverIntervalSeconds = 10 * 60;
- private String mArchiveDirectory = null;
- private int mArchiveFileCount = 10;
- private DebugTracer mLogger = null;
+ public LocalFileLogBuffer(DebugTracer tracer) {
+ mLogger = tracer;
+ }
- private Writer mWriter = null;
- private String mBufferFilename = null;
- private long mNextRolloverTime = 0;
- private long mNextFlushTime = 0;
- private int mFileOpenRetryIntervalInMs = 60 * 1000;
- private long mNextFileOpenRetryTime = 0;
-
- private DestinationDispatcherThread<T> mDispatcherThread = null;
-
- public LocalFileLogBuffer(DebugTracer tracer) {
- mLogger = tracer;
- }
-
- public String getDirectory() {
- return mDirectory;
- }
-
- public void setDirectory(String directory) {
- mDirectory = directory;
- }
-
- public String getFile() {
- return mFile;
- }
-
- public void setFile(String file) {
- mFile = file;
- }
-
- public int getFileBufferSizeBytes() {
- return mFileBufferSizeBytes;
- }
-
- public void setFileBufferSizeBytes(int fileBufferSizeBytes) {
- mFileBufferSizeBytes = fileBufferSizeBytes;
- }
-
- public int getFlushIntervalSeconds() {
- return mFlushIntervalSeconds;
- }
-
- public void setFlushIntervalSeconds(int flushIntervalSeconds) {
- mFlushIntervalSeconds = flushIntervalSeconds;
- }
-
- public String getEncoding() {
- return mEncoding;
- }
-
- public void setEncoding(String encoding) {
- mEncoding = encoding;
- }
-
- public boolean getIsAppend() {
- return mIsAppend;
- }
-
- public void setIsAppend(boolean isAppend) {
- mIsAppend = isAppend;
- }
-
- public int getRolloverIntervalSeconds() {
- return mRolloverIntervalSeconds;
- }
-
- public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
- mRolloverIntervalSeconds = rolloverIntervalSeconds;
- }
-
- public String getArchiveDirectory() {
- return mArchiveDirectory;
- }
-
- public void setArchiveDirectory(String archiveDirectory) {
- mArchiveDirectory = archiveDirectory;
- }
-
- public int getArchiveFileCount() {
- return mArchiveFileCount;
- }
-
- public void setArchiveFileCount(int archiveFileCount) {
- mArchiveFileCount = archiveFileCount;
- }
-
-
- @Override
- public void start(LogDestination<T> destination) {
- mLogger.debug("==> LocalFileLogBuffer.start()");
-
- mDispatcherThread = new DestinationDispatcherThread<T>(this, destination, mLogger);
-
- mDispatcherThread.setDaemon(true);
-
- mDispatcherThread.start();
-
- mLogger.debug("<== LocalFileLogBuffer.start()");
- }
-
- @Override
- public void stop() {
- mLogger.debug("==> LocalFileLogBuffer.stop()");
-
- DestinationDispatcherThread<T> dispatcherThread = mDispatcherThread;
- mDispatcherThread = null;
-
- if(dispatcherThread != null && dispatcherThread.isAlive()) {
- dispatcherThread.stopThread();
-
- try {
- dispatcherThread.join();
- } catch (InterruptedException e) {
- mLogger.warn("LocalFileLogBuffer.stop(): failed in waiting for DispatcherThread", e);
- }
- }
+ public String getDirectory() {
+ return mDirectory;
+ }
- closeFile();
-
- mLogger.debug("<== LocalFileLogBuffer.stop()");
- }
-
- @Override
- public boolean isAvailable() {
- return mWriter != null;
- }
-
- @Override
- public boolean add(T log) {
- boolean ret = false;
-
- String msg = MiscUtil.stringify(log);
-
- if(msg.contains(MiscUtil.LINE_SEPARATOR)) {
- msg = msg.replace(MiscUtil.LINE_SEPARATOR, MiscUtil.ESCAPE_STR + MiscUtil.LINE_SEPARATOR);
- }
+ public void setDirectory(String directory) {
+ mDirectory = directory;
+ }
- synchronized(this) {
- checkFileStatus();
-
- Writer writer = mWriter;
-
- if(writer != null) {
- try {
- writer.write(msg + MiscUtil.LINE_SEPARATOR);
-
- if(mFileBufferSizeBytes == 0) {
- writer.flush();
- }
-
- ret = true;
- } catch(IOException excp) {
- mLogger.warn("LocalFileLogBuffer.add(): write failed", excp);
+ public String getFile() {
+ return mFile;
+ }
- closeFile();
- }
- }
- }
-
- return ret;
- }
-
- @Override
- public boolean isEmpty() {
- return mDispatcherThread == null || mDispatcherThread.isIdle();
- }
-
- private synchronized void openFile() {
- mLogger.debug("==> LocalFileLogBuffer.openFile()");
-
- long now = System.currentTimeMillis();
-
- closeFile();
-
- if(mNextFileOpenRetryTime <= now) {
- try {
- mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
-
- long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
-
- mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
-
- MiscUtil.createParents(new File(mBufferFilename));
-
- FileOutputStream ostream = null;
- try {
- ostream = new FileOutputStream(mBufferFilename, mIsAppend);
- } catch(Exception excp) {
- mLogger.warn("LocalFileLogBuffer.openFile(): failed to open file " + mBufferFilename, excp);
- }
-
- if(ostream != null) {
- mWriter = createWriter(ostream);
- mLogger.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
- mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L);
- } else {
- mLogger.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
- mBufferFilename = null;
- }
- } finally {
- if(mWriter == null) {
- mNextFileOpenRetryTime = now + mFileOpenRetryIntervalInMs;
- }
- }
- }
-
- mLogger.debug("<== LocalFileLogBuffer.openFile()");
- }
-
- private synchronized void closeFile() {
- mLogger.debug("==> LocalFileLogBuffer.closeFile()");
-
- Writer writer = mWriter;
-
- mWriter = null;
-
- if(writer != null) {
- try {
- writer.flush();
- writer.close();
- } catch(IOException excp) {
- mLogger.warn("LocalFileLogBuffer: failed to close file " + mBufferFilename, excp);
- }
-
- if(mDispatcherThread != null) {
- mDispatcherThread.addLogfile(mBufferFilename);
- }
- }
-
- mLogger.debug("<== LocalFileLogBuffer.closeFile()");
- }
-
- private void rollover() {
- mLogger.debug("==> LocalFileLogBuffer.rollover()");
-
- closeFile();
-
- openFile();
-
- mLogger.debug("<== LocalFileLogBuffer.rollover()");
- }
-
- private void checkFileStatus() {
- long now = System.currentTimeMillis();
-
- if(now > mNextRolloverTime) {
- rollover();
- } else if(mWriter == null) {
- openFile();
- } else if(now > mNextFlushTime) {
- try {
- mNextFlushTime = now + (mFlushIntervalSeconds * 1000L);
-
- mWriter.flush();
- } catch (IOException excp) {
- mLogger.warn("LocalFileLogBuffer: failed to flush to file " + mBufferFilename, excp);
- }
- }
- }
-
- private Writer createWriter(OutputStream os ) {
- Writer writer = null;
-
- if(os != null) {
- if(mEncoding != null) {
- try {
- writer = new OutputStreamWriter(os, mEncoding);
- } catch(UnsupportedEncodingException excp) {
- mLogger.warn("LocalFileLogBuffer: failed to create output writer for file " + mBufferFilename, excp);
- }
- }
-
- if(writer == null) {
- writer = new OutputStreamWriter(os);
- }
-
- if(mFileBufferSizeBytes > 0 && writer != null) {
- writer = new BufferedWriter(writer, mFileBufferSizeBytes);
- }
- }
-
- return writer;
- }
-
- boolean isCurrentFilename(String filename) {
- return filename != null && filename.equals(mBufferFilename);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
-
- sb.append("LocalFileLogBuffer {");
- sb.append("Directory=").append(mDirectory).append("; ");
- sb.append("File=").append(mFile).append("; ");
- sb.append("RolloverIntervaSeconds=").append(mRolloverIntervalSeconds).append("; ");
- sb.append("ArchiveDirectory=").append(mArchiveDirectory).append("; ");
- sb.append("ArchiveFileCount=").append(mArchiveFileCount);
- sb.append("}");
-
- return sb.toString();
- }
-
-}
+ public void setFile(String file) {
+ mFile = file;
+ }
-class DestinationDispatcherThread<T> extends Thread {
- private TreeSet<String> mCompletedLogfiles = new TreeSet<String>();
- private boolean mStopThread = false;
- private LocalFileLogBuffer<T> mFileLogBuffer = null;
- private LogDestination<T> mDestination = null;
- private DebugTracer mLogger = null;
-
- private String mCurrentLogfile = null;
-
- public DestinationDispatcherThread(LocalFileLogBuffer<T> fileLogBuffer, LogDestination<T> destination, DebugTracer tracer) {
- super(DestinationDispatcherThread.class.getSimpleName() + "-" + System.currentTimeMillis());
-
- mLogger = tracer;
-
- mFileLogBuffer = fileLogBuffer;
- mDestination = destination;
-
- setDaemon(true);
- }
-
- public void addLogfile(String filename) {
- mLogger.debug("==> DestinationDispatcherThread.addLogfile(" + filename + ")");
-
- if(filename != null) {
- synchronized(mCompletedLogfiles) {
- mCompletedLogfiles.add(filename);
- mCompletedLogfiles.notifyAll();
- }
- }
-
- mLogger.debug("<== DestinationDispatcherThread.addLogfile(" + filename + ")");
- }
-
- public void stopThread() {
- mStopThread = true;
- }
-
- public boolean isIdle() {
- synchronized(mCompletedLogfiles) {
- return mCompletedLogfiles.isEmpty() && mCurrentLogfile == null;
- }
- }
-
- @Override
- public void run() {
- UserGroupInformation loginUser = null;
-
- try {
- loginUser = UserGroupInformation.getLoginUser();
- } catch (IOException excp) {
- mLogger.error("DestinationDispatcherThread.run(): failed to get login user details. Audit files will not be sent to HDFS destination", excp);
- }
-
- if(loginUser == null) {
- mLogger.error("DestinationDispatcherThread.run(): failed to get login user. Audit files will not be sent to HDFS destination");
-
- return;
- }
-
- try {
- loginUser.doAs((PrivilegedExceptionAction<Integer>) () -> {
- doRun();
-
- return 0;
- });
- } catch (Exception excp) {
- mLogger.error("DestinationDispatcherThread.run(): failed", excp);
- }
- }
-
- private void doRun() {
- init();
-
- mDestination.start();
-
- long pollIntervalInMs = 1000L;
-
- while(! mStopThread) {
- synchronized(mCompletedLogfiles) {
- while(mCompletedLogfiles.isEmpty() && !mStopThread) {
- try {
- mCompletedLogfiles.wait(pollIntervalInMs);
- } catch(InterruptedException excp) {
- throw new RuntimeException("DestinationDispatcherThread.run(): failed to wait for log file", excp);
- }
- }
-
- mCurrentLogfile = mCompletedLogfiles.pollFirst();
- }
-
- if(mCurrentLogfile != null) {
- sendCurrentFile();
- }
- }
-
- mDestination.stop();
- }
-
- private void init() {
- mLogger.debug("==> DestinationDispatcherThread.init()");
-
- String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory(), 0);
-
- if(dirName != null) {
- File directory = new File(dirName);
-
- if(directory.exists() && directory.isDirectory()) {
- File[] files = directory.listFiles();
-
- if(files != null) {
- for(File file : files) {
- if(file.exists() && file.isFile() && file.canRead()) {
- String filename = file.getAbsolutePath();
- if(! mFileLogBuffer.isCurrentFilename(filename)) {
- addLogfile(filename);
- }
- }
- }
- }
- }
- }
-
- mLogger.debug("<== DestinationDispatcherThread.init()");
- }
-
- private boolean sendCurrentFile() {
- mLogger.debug("==> DestinationDispatcherThread.sendCurrentFile()");
-
- boolean ret = false;
-
- long destinationPollIntervalInMs = 1000L;
-
- BufferedReader reader = openCurrentFile();
- try {
- while(!mStopThread) {
- String log = getNextStringifiedLog(reader);
-
- if(log == null) { // reached end-of-file
- ret = true;
-
- break;
- }
-
- try {
- // loop until log is sent successfully
- while(!mStopThread && !mDestination.sendStringified(log)) {
- try {
- Thread.sleep(destinationPollIntervalInMs);
- } catch(InterruptedException excp) {
- throw new RuntimeException("LocalFileLogBuffer.sendCurrentFile(" + mCurrentLogfile + "): failed while waiting for destination to be available", excp);
- }
- }
- } catch ( AuditMessageException msgError) {
- mLogger.error("Error in log message:" + log);
- //If there is error in log message, then it will be skipped
- }
- }
- } finally {
- closeCurrentFile(reader);
- }
-
- if(!mStopThread) {
- mDestination.flush();
- archiveCurrentFile();
- }
-
- mLogger.debug("<== DestinationDispatcherThread.sendCurrentFile()");
-
- return ret;
- }
-
- private String getNextStringifiedLog(BufferedReader mReader) {
- String log = null;
-
- if(mReader != null) {
- try {
- while(true) {
- String line = mReader.readLine();
-
- if(line == null) { // reached end-of-file
- break;
- }
-
- if(line.endsWith(MiscUtil.ESCAPE_STR)) {
- line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
-
- if(log == null) {
- log = line;
- } else {
- log += MiscUtil.LINE_SEPARATOR;
- log += line;
- }
-
- continue;
- } else {
- if(log == null) {
- log = line;
- } else {
- log += line;
- }
- break;
- }
- }
- } catch (IOException excp) {
- mLogger.warn("getNextStringifiedLog.getNextLog(): failed to read from file " + mCurrentLogfile, excp);
- }
- }
-
- return log;
- }
-
- private BufferedReader openCurrentFile() {
- mLogger.debug("==> openCurrentFile(" + mCurrentLogfile + ")");
- BufferedReader mReader = null;
-
- if(mCurrentLogfile != null) {
- try {
- FileInputStream inStr = new FileInputStream(mCurrentLogfile);
-
- InputStreamReader strReader = createReader(inStr);
-
- if(strReader != null) {
- mReader = new BufferedReader(strReader);
- }
- } catch(FileNotFoundException excp) {
- mLogger.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp);
- }
- }
-
- mLogger.debug("<== openCurrentFile(" + mCurrentLogfile + ")");
- return mReader;
- }
-
- private void closeCurrentFile(BufferedReader mReader) {
- mLogger.debug("==> closeCurrentFile(" + mCurrentLogfile + ")");
-
- if(mReader != null) {
- try {
- mReader.close();
- } catch(IOException excp) {
- // ignore
- }
- }
-
- mLogger.debug("<== closeCurrentFile(" + mCurrentLogfile + ")");
- }
-
- private void archiveCurrentFile() {
- if(mCurrentLogfile != null) {
- File logFile = new File(mCurrentLogfile);
- String archiveDirName = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory(), 0);
- String archiveFilename = archiveDirName + File.separator +logFile.getName();
-
- try {
- if(logFile.exists()) {
- File archiveFile = new File(archiveFilename);
-
- MiscUtil.createParents(archiveFile);
-
- if(! logFile.renameTo(archiveFile)) {
- // TODO: renameTo() does not work in all cases. in case of failure, copy the file contents to the destination and delete the file
- mLogger.warn("archiving failed to move file: " + mCurrentLogfile + " ==> " + archiveFilename);
- }
-
- File archiveDir = new File(archiveDirName);
- File[] files = archiveDir.listFiles(new FileFilter() {
- @Override
- public boolean accept(File f) {
- return f.isFile();
- }
- });
-
- int numOfFilesToDelete = files == null ? 0 : (files.length - mFileLogBuffer.getArchiveFileCount());
-
- if(numOfFilesToDelete > 0) {
- Arrays.sort(files, new Comparator<File>() {
- @Override
- public int compare(File f1, File f2) {
- return (int)(f1.lastModified() - f2.lastModified());
- }
- });
-
- for(int i = 0; i < numOfFilesToDelete; i++) {
- if(! files[i].delete()) {
- mLogger.warn("archiving failed to delete file: " + files[i].getAbsolutePath());
- }
- }
- }
- }
- } catch(Exception excp) {
- mLogger.warn("archiveCurrentFile(): faile to move " + mCurrentLogfile + " to archive location " + archiveFilename, excp);
- }
- }
- mCurrentLogfile = null;
- }
-
- private InputStreamReader createReader(InputStream iStr) {
- InputStreamReader reader = null;
-
- if(iStr != null) {
- String encoding = mFileLogBuffer.getEncoding();
-
- if(encoding != null) {
- try {
- reader = new InputStreamReader(iStr, encoding);
- } catch(UnsupportedEncodingException excp) {
- mLogger.warn("createReader(): failed to create input reader.", excp);
- }
- }
-
- if(reader == null) {
- reader = new InputStreamReader(iStr);
- }
- }
-
- return reader;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
-
- sb.append("DestinationDispatcherThread {");
- sb.append("ThreadName=").append(this.getName()).append("; ");
- sb.append("CompletedLogfiles.size()=").append(mCompletedLogfiles.size()).append("; ");
- sb.append("StopThread=").append(mStopThread).append("; ");
- sb.append("CurrentLogfile=").append(mCurrentLogfile);
- sb.append("}");
-
- return sb.toString();
- }
-}
+ public int getFileBufferSizeBytes() {
+ return mFileBufferSizeBytes;
+ }
+
+ public void setFileBufferSizeBytes(int fileBufferSizeBytes) {
+ mFileBufferSizeBytes = fileBufferSizeBytes;
+ }
+
+ public int getFlushIntervalSeconds() {
+ return mFlushIntervalSeconds;
+ }
+
+ public void setFlushIntervalSeconds(int flushIntervalSeconds) {
+ mFlushIntervalSeconds = flushIntervalSeconds;
+ }
+
+ public String getEncoding() {
+ return mEncoding;
+ }
+
+ public void setEncoding(String encoding) {
+ mEncoding = encoding;
+ }
+
+ public boolean getIsAppend() {
+ return mIsAppend;
+ }
+
+ public void setIsAppend(boolean isAppend) {
+ mIsAppend = isAppend;
+ }
+
+ public int getRolloverIntervalSeconds() {
+ return mRolloverIntervalSeconds;
+ }
+
+ public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
+ mRolloverIntervalSeconds = rolloverIntervalSeconds;
+ }
+
+ public String getArchiveDirectory() {
+ return mArchiveDirectory;
+ }
+
+ public void setArchiveDirectory(String archiveDirectory) {
+ mArchiveDirectory = archiveDirectory;
+ }
+
+ public int getArchiveFileCount() {
+ return mArchiveFileCount;
+ }
+
+ public void setArchiveFileCount(int archiveFileCount) {
+ mArchiveFileCount = archiveFileCount;
+ }
+
+ @Override
+ public void start(LogDestination<T> destination) {
+ mLogger.debug("==> LocalFileLogBuffer.start()");
+
+ mDispatcherThread = new DestinationDispatcherThread<>(this, destination, mLogger);
+
+ mDispatcherThread.setDaemon(true);
+
+ mDispatcherThread.start();
+
+ mLogger.debug("<== LocalFileLogBuffer.start()");
+ }
+
+ @Override
+ public void stop() {
+ mLogger.debug("==> LocalFileLogBuffer.stop()");
+
+ DestinationDispatcherThread<T> dispatcherThread = mDispatcherThread;
+
+ mDispatcherThread = null;
+
+ if (dispatcherThread != null && dispatcherThread.isAlive()) {
+ dispatcherThread.stopThread();
+
+ try {
+ dispatcherThread.join();
+ } catch (InterruptedException e) {
+ mLogger.warn("LocalFileLogBuffer.stop(): failed in waiting for DispatcherThread", e);
+ }
+ }
+
+ closeFile();
+
+ mLogger.debug("<== LocalFileLogBuffer.stop()");
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return mWriter != null;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return mDispatcherThread == null || mDispatcherThread.isIdle();
+ }
+
+ @Override
+ public boolean add(T log) {
+ boolean ret = false;
+ String msg = MiscUtil.stringify(log);
+
+ if (msg.contains(MiscUtil.LINE_SEPARATOR)) {
+ msg = msg.replace(MiscUtil.LINE_SEPARATOR, MiscUtil.ESCAPE_STR + MiscUtil.LINE_SEPARATOR);
+ }
+
+ synchronized (this) {
+ checkFileStatus();
+ Writer writer = mWriter;
+
+ if (writer != null) {
+ try {
+ writer.write(msg + MiscUtil.LINE_SEPARATOR);
+
+ if (mFileBufferSizeBytes == 0) {
+ writer.flush();
+ }
+
+ ret = true;
+ } catch (IOException excp) {
+ mLogger.warn("LocalFileLogBuffer.add(): write failed", excp);
+
+ closeFile();
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("LocalFileLogBuffer {");
+ sb.append("Directory=").append(mDirectory).append("; ");
+ sb.append("File=").append(mFile).append("; ");
+ sb.append("RolloverIntervaSeconds=").append(mRolloverIntervalSeconds).append("; ");
+ sb.append("ArchiveDirectory=").append(mArchiveDirectory).append("; ");
+ sb.append("ArchiveFileCount=").append(mArchiveFileCount);
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+ boolean isCurrentFilename(String filename) {
+ return filename != null && filename.equals(mBufferFilename);
+ }
+
+ private synchronized void openFile() {
+ mLogger.debug("==> LocalFileLogBuffer.openFile()");
+
+ long now = System.currentTimeMillis();
+
+ closeFile();
+
+ if (mNextFileOpenRetryTime <= now) {
+ try {
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
+
+ long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
+
+ mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
+
+ MiscUtil.createParents(new File(mBufferFilename));
+
+ FileOutputStream ostream = null;
+
+ try {
+ ostream = new FileOutputStream(mBufferFilename, mIsAppend);
+ } catch (Exception excp) {
+ mLogger.warn("LocalFileLogBuffer.openFile(): failed to open file " + mBufferFilename, excp);
+ }
+
+ if (ostream != null) {
+ mWriter = createWriter(ostream);
+
+ mLogger.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
+
+ mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L);
+ } else {
+ mLogger.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
+
+ mBufferFilename = null;
+ }
+ } finally {
+ if (mWriter == null) {
+ mNextFileOpenRetryTime = now + mFileOpenRetryIntervalInMs;
+ }
+ }
+ }
+
+ mLogger.debug("<== LocalFileLogBuffer.openFile()");
+ }
+
+ private synchronized void closeFile() {
+ mLogger.debug("==> LocalFileLogBuffer.closeFile()");
+
+ Writer writer = mWriter;
+
+ mWriter = null;
+
+ if (writer != null) {
+ try {
+ writer.flush();
+ writer.close();
+ } catch (IOException excp) {
+ mLogger.warn("LocalFileLogBuffer: failed to close file " + mBufferFilename, excp);
+ }
+
+ if (mDispatcherThread != null) {
+ mDispatcherThread.addLogfile(mBufferFilename);
+ }
+ }
+
+ mLogger.debug("<== LocalFileLogBuffer.closeFile()");
+ }
+
+ private void rollover() {
+ mLogger.debug("==> LocalFileLogBuffer.rollover()");
+
+ closeFile();
+
+ openFile();
+
+ mLogger.debug("<== LocalFileLogBuffer.rollover()");
+ }
+
+ private void checkFileStatus() {
+ long now = System.currentTimeMillis();
+
+ if (now > mNextRolloverTime) {
+ rollover();
+ } else if (mWriter == null) {
+ openFile();
+ } else if (now > mNextFlushTime) {
+ try {
+ mNextFlushTime = now + (mFlushIntervalSeconds * 1000L);
+
+ mWriter.flush();
+ } catch (IOException excp) {
+ mLogger.warn("LocalFileLogBuffer: failed to flush to file " + mBufferFilename, excp);
+ }
+ }
+ }
+
+ private Writer createWriter(OutputStream os) {
+ Writer writer = null;
+
+ if (os != null) {
+ if (mEncoding != null) {
+ try {
+ writer = new OutputStreamWriter(os, mEncoding);
+ } catch (UnsupportedEncodingException excp) {
+ mLogger.warn("LocalFileLogBuffer: failed to create output writer for file " + mBufferFilename, excp);
+ }
+ }
+
+ if (writer == null) {
+ writer = new OutputStreamWriter(os);
+ }
+
+ if (mFileBufferSizeBytes > 0) {
+ writer = new BufferedWriter(writer, mFileBufferSizeBytes);
+ }
+ }
+
+ return writer;
+ }
+
+ static class DestinationDispatcherThread<T> extends Thread {
+ private final TreeSet<String> mCompletedLogfiles = new TreeSet<>();
+ private final LocalFileLogBuffer<T> mFileLogBuffer;
+ private final LogDestination<T> mDestination;
+ private final DebugTracer mLogger;
+ private String mCurrentLogfile;
+ private boolean mStopThread;
+
+ public DestinationDispatcherThread(LocalFileLogBuffer<T> fileLogBuffer, LogDestination<T> destination, DebugTracer tracer) {
+ super(DestinationDispatcherThread.class.getSimpleName() + "-" + System.currentTimeMillis());
+
+ mLogger = tracer;
+ mFileLogBuffer = fileLogBuffer;
+ mDestination = destination;
+
+ setDaemon(true);
+ }
+
+ public void addLogfile(String filename) {
+ mLogger.debug("==> DestinationDispatcherThread.addLogfile(" + filename + ")");
+
+ if (filename != null) {
+ synchronized (mCompletedLogfiles) {
+ mCompletedLogfiles.add(filename);
+ mCompletedLogfiles.notifyAll();
+ }
+ }
+
+ mLogger.debug("<== DestinationDispatcherThread.addLogfile(" + filename + ")");
+ }
+
+ public void stopThread() {
+ mStopThread = true;
+ }
+
+ public boolean isIdle() {
+ synchronized (mCompletedLogfiles) {
+ return mCompletedLogfiles.isEmpty() && mCurrentLogfile == null;
+ }
+ }
+
+ @Override
+ public void run() {
+ UserGroupInformation loginUser = null;
+
+ try {
+ loginUser = UserGroupInformation.getLoginUser();
+ } catch (IOException excp) {
+ mLogger.error("DestinationDispatcherThread.run(): failed to get login user details. Audit files will not be sent to HDFS destination", excp);
+ }
+
+ if (loginUser == null) {
+ mLogger.error("DestinationDispatcherThread.run(): failed to get login user. Audit files will not be sent to HDFS destination");
+
+ return;
+ }
+
+ try {
+ loginUser.doAs((PrivilegedExceptionAction<Integer>) () -> {
+ doRun();
+
+ return 0;
+ });
+ } catch (Exception excp) {
+ mLogger.error("DestinationDispatcherThread.run(): failed", excp);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("DestinationDispatcherThread {");
+ sb.append("ThreadName=").append(this.getName()).append("; ");
+ sb.append("CompletedLogfiles.size()=").append(mCompletedLogfiles.size()).append("; ");
+ sb.append("StopThread=").append(mStopThread).append("; ");
+ sb.append("CurrentLogfile=").append(mCurrentLogfile);
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+ private void doRun() {
+ init();
+
+ mDestination.start();
+
+ long pollIntervalInMs = 1000L;
+
+ while (!mStopThread) {
+ synchronized (mCompletedLogfiles) {
+ while (mCompletedLogfiles.isEmpty() && !mStopThread) {
+ try {
+ mCompletedLogfiles.wait(pollIntervalInMs);
+ } catch (InterruptedException excp) {
+ throw new RuntimeException("DestinationDispatcherThread.run(): failed to wait for log file", excp);
+ }
+ }
+
+ mCurrentLogfile = mCompletedLogfiles.pollFirst();
+ }
+
+ if (mCurrentLogfile != null) {
+ sendCurrentFile();
+ }
+ }
+
+ mDestination.stop();
+ }
+
+ private void init() {
+ mLogger.debug("==> DestinationDispatcherThread.init()");
+
+ String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory(), 0);
+
+ if (dirName != null) {
+ File directory = new File(dirName);
+
+ if (directory.exists() && directory.isDirectory()) {
+ File[] files = directory.listFiles();
+
+ if (files != null) {
+ for (File file : files) {
+ if (file.exists() && file.isFile() && file.canRead()) {
+ String filename = file.getAbsolutePath();
+
+ if (!mFileLogBuffer.isCurrentFilename(filename)) {
+ addLogfile(filename);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ mLogger.debug("<== DestinationDispatcherThread.init()");
+ }
+
+ private boolean sendCurrentFile() {
+ mLogger.debug("==> DestinationDispatcherThread.sendCurrentFile()");
+
+ boolean ret = false;
+ long destinationPollIntervalInMs = 1000L;
+
+ BufferedReader reader = openCurrentFile();
+ try {
+ while (!mStopThread) {
+ String log = getNextStringifiedLog(reader);
+
+ if (log == null) { // reached end-of-file
+ ret = true;
+
+ break;
+ }
+
+ try {
+ // loop until log is sent successfully
+ while (!mStopThread && !mDestination.sendStringified(log)) {
+ try {
+ Thread.sleep(destinationPollIntervalInMs);
+ } catch (InterruptedException excp) {
+ throw new RuntimeException("LocalFileLogBuffer.sendCurrentFile(" + mCurrentLogfile + "): failed while waiting for destination to be available", excp);
+ }
+ }
+ } catch (AuditMessageException msgError) {
+ mLogger.error("Error in log message:" + log);
+ //If there is error in log message, then it will be skipped
+ }
+ }
+ } finally {
+ closeCurrentFile(reader);
+ }
+
+ if (!mStopThread) {
+ mDestination.flush();
+ archiveCurrentFile();
+ }
+
+ mLogger.debug("<== DestinationDispatcherThread.sendCurrentFile()");
+
+ return ret;
+ }
+
+ private String getNextStringifiedLog(BufferedReader mReader) {
+ String log = null;
+
+ if (mReader != null) {
+ try {
+ while (true) {
+ String line = mReader.readLine();
+
+ if (line == null) { // reached end-of-file
+ break;
+ }
+
+ if (line.endsWith(MiscUtil.ESCAPE_STR)) {
+ line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
+
+ if (log == null) {
+ log = line;
+ } else {
+ log += MiscUtil.LINE_SEPARATOR;
+ log += line;
+ }
+ } else {
+ if (log == null) {
+ log = line;
+ } else {
+ log += line;
+ }
+ break;
+ }
+ }
+ } catch (IOException excp) {
+ mLogger.warn("getNextStringifiedLog.getNextLog(): failed to read from file " + mCurrentLogfile, excp);
+ }
+ }
+
+ return log;
+ }
+
+ private BufferedReader openCurrentFile() {
+ mLogger.debug("==> openCurrentFile(" + mCurrentLogfile + ")");
+
+ BufferedReader mReader = null;
+
+ if (mCurrentLogfile != null) {
+ try {
+ FileInputStream inStr = new FileInputStream(mCurrentLogfile);
+
+ InputStreamReader strReader = createReader(inStr);
+
+ if (strReader != null) {
+ mReader = new BufferedReader(strReader);
+ }
+ } catch (FileNotFoundException excp) {
+ mLogger.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp);
+ }
+ }
+
+ mLogger.debug("<== openCurrentFile(" + mCurrentLogfile + ")");
+
+ return mReader;
+ }
+
+ private void closeCurrentFile(BufferedReader mReader) {
+ mLogger.debug("==> closeCurrentFile(" + mCurrentLogfile + ")");
+
+ if (mReader != null) {
+ try {
+ mReader.close();
+ } catch (IOException excp) {
+ // ignore
+ }
+ }
+
+ mLogger.debug("<== closeCurrentFile(" + mCurrentLogfile + ")");
+ }
+
+ private void archiveCurrentFile() {
+ if (mCurrentLogfile != null) {
+ File logFile = new File(mCurrentLogfile);
+ String archiveDirName = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory(), 0);
+ String archiveFilename = archiveDirName + File.separator + logFile.getName();
+
+ try {
+ if (logFile.exists()) {
+ File archiveFile = new File(archiveFilename);
+
+ MiscUtil.createParents(archiveFile);
+
+ if (!logFile.renameTo(archiveFile)) {
+ // TODO: renameTo() does not work in all cases. in case of failure, copy the file contents to the destination and delete the file
+ mLogger.warn("archiving failed to move file: " + mCurrentLogfile + " ==> " + archiveFilename);
+ }
+
+ File archiveDir = new File(archiveDirName);
+ File[] files = archiveDir.listFiles(File::isFile);
+ int numOfFilesToDelete = files == null ? 0 : (files.length - mFileLogBuffer.getArchiveFileCount());
+
+ if (numOfFilesToDelete > 0) {
+ Arrays.sort(files, (f1, f2) -> (int) (f1.lastModified() - f2.lastModified()));
+
+ for (int i = 0; i < numOfFilesToDelete; i++) {
+ if (!files[i].delete()) {
+ mLogger.warn("archiving failed to delete file: " + files[i].getAbsolutePath());
+ }
+ }
+ }
+ }
+ } catch (Exception excp) {
+ mLogger.warn("archiveCurrentFile(): faile to move " + mCurrentLogfile + " to archive location " + archiveFilename, excp);
+ }
+ }
+
+ mCurrentLogfile = null;
+ }
+
+ private InputStreamReader createReader(InputStream iStr) {
+ InputStreamReader reader = null;
+
+ if (iStr != null) {
+ String encoding = mFileLogBuffer.getEncoding();
+
+ if (encoding != null) {
+ try {
+ reader = new InputStreamReader(iStr, encoding);
+ } catch (UnsupportedEncodingException excp) {
+ mLogger.warn("createReader(): failed to create input reader.", excp);
+ }
+ }
+
+ if (reader == null) {
+ reader = new InputStreamReader(iStr);
+ }
+ }
+
+ return reader;
+ }
+ }
+}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
index 0a874ef7d..350f77804 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,83 +18,81 @@
package org.apache.ranger.audit.provider;
-import java.util.Collection;
-import java.util.Properties;
-
import org.apache.ranger.audit.destination.AuditDestination;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
+import java.util.Properties;
public class Log4jAuditProvider extends AuditDestination {
+ private static final Logger LOG = LoggerFactory.getLogger(Log4jAuditProvider.class);
+ private static final Logger AUDITLOG = LoggerFactory.getLogger("xaaudit." + Log4jAuditProvider.class.getName());
+
+ public static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async";
+ public static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size";
+ public static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms";
+
+ public Log4jAuditProvider() {
+ LOG.info("Log4jAuditProvider: creating..");
+ }
+
+ @Override
+ public boolean log(AuditEventBase event) {
+ if (!AUDITLOG.isInfoEnabled()) {
+ return true;
+ }
+
+ if (event != null) {
+ String eventStr = MiscUtil.stringify(event);
+
+ AUDITLOG.info(eventStr);
+ }
+
+ return true;
+ }
+
+ @Override
+ public boolean logJSON(String event) {
+ AuditEventBase eventObj = MiscUtil.fromJson(event, AuthzAuditEvent.class);
+
+ return log(eventObj);
+ }
+
+ @Override
+ public boolean logJSON(Collection<String> events) {
+ for (String event : events) {
+ logJSON(event);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void init(Properties props) {
+ LOG.info("Log4jAuditProvider.init()");
+
+ super.init(props);
+ }
+
+ @Override
+ public boolean log(Collection<AuditEventBase> events) {
+ for (AuditEventBase event : events) {
+ log(event);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void start() {
+ // intentionally left empty
+ }
- private static final Logger LOG = LoggerFactory.getLogger(Log4jAuditProvider.class);
- private static final Logger AUDITLOG = LoggerFactory.getLogger("xaaudit." + Log4jAuditProvider.class.getName());
-
- public static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async";
- public static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size";
- public static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms";
-
-
- public Log4jAuditProvider() {
- LOG.info("Log4jAuditProvider: creating..");
- }
-
- @Override
- public void init(Properties props) {
- LOG.info("Log4jAuditProvider.init()");
-
- super.init(props);
- }
-
- @Override
- public boolean log(AuditEventBase event) {
- if(! AUDITLOG.isInfoEnabled())
- return true;
-
- if(event != null) {
- String eventStr = MiscUtil.stringify(event);
- AUDITLOG.info(eventStr);
- }
- return true;
- }
-
- @Override
- public boolean log(Collection<AuditEventBase> events) {
- for (AuditEventBase event : events) {
- log(event);
- }
- return true;
- }
-
- @Override
- public boolean logJSON(String event) {
- AuditEventBase eventObj = MiscUtil.fromJson(event,
- AuthzAuditEvent.class);
- return log(eventObj);
- }
-
- @Override
- public boolean logJSON(Collection<String> events) {
- for (String event : events) {
- logJSON(event);
- }
- return true;
- }
-
- @Override
- public void start() {
- // intentionally left empty
- }
-
- @Override
- public void stop() {
- // intentionally left empty
- }
-
-
-
-
+ @Override
+ public void stop() {
+ // intentionally left empty
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jTracer.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jTracer.java
index cd5befcd2..d246cd2cb 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jTracer.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jTracer.java
@@ -19,41 +19,41 @@ package org.apache.ranger.audit.provider;
import org.slf4j.Logger;
public class Log4jTracer implements DebugTracer {
- private Logger mLogger = null;
+ private final Logger mLogger;
- public Log4jTracer(Logger logger) {
- mLogger = logger;
- }
+ public Log4jTracer(Logger logger) {
+ mLogger = logger;
+ }
- public void debug(String msg) {
- mLogger.debug(msg);
- }
+ public void debug(String msg) {
+ mLogger.debug(msg);
+ }
- public void debug(String msg, Throwable excp) {
- mLogger.debug(msg, excp);
- }
+ public void debug(String msg, Throwable excp) {
+ mLogger.debug(msg, excp);
+ }
- public void info(String msg) {
- mLogger.info(msg);
- }
+ public void info(String msg) {
+ mLogger.info(msg);
+ }
- public void info(String msg, Throwable excp) {
- mLogger.info(msg, excp);
- }
+ public void info(String msg, Throwable excp) {
+ mLogger.info(msg, excp);
+ }
- public void warn(String msg) {
- mLogger.warn(msg);
- }
+ public void warn(String msg) {
+ mLogger.warn(msg);
+ }
- public void warn(String msg, Throwable excp) {
- mLogger.warn(msg, excp);
- }
+ public void warn(String msg, Throwable excp) {
+ mLogger.warn(msg, excp);
+ }
- public void error(String msg) {
- mLogger.error(msg);
- }
+ public void error(String msg) {
+ mLogger.error(msg);
+ }
- public void error(String msg, Throwable excp) {
- mLogger.error(msg, excp);
- }
+ public void error(String msg, Throwable excp) {
+ mLogger.error(msg, excp);
+ }
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogBuffer.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogBuffer.java
index d6646924f..ca6d6d512 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogBuffer.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogBuffer.java
@@ -18,15 +18,14 @@
*/
package org.apache.ranger.audit.provider;
-
public interface LogBuffer<T> {
- void start(LogDestination<T> destination);
+ void start(LogDestination<T> destination);
- void stop();
+ void stop();
- boolean isAvailable();
+ boolean isAvailable();
- boolean isEmpty();
+ boolean isEmpty();
- boolean add(T log);
+ boolean add(T log);
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogDestination.java
index a9c3af938..4c53662a8 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/LogDestination.java
@@ -21,26 +21,26 @@ package org.apache.ranger.audit.provider;
import org.apache.ranger.audit.model.AuditEventBase;
public interface LogDestination<T> {
- void start();
+ void start();
- void stop();
+ void stop();
- boolean isAvailable();
+ boolean isAvailable();
- boolean send(AuditEventBase log) throws AuditMessageException;
+ boolean send(AuditEventBase log) throws AuditMessageException;
- boolean send(AuditEventBase[] logs) throws AuditMessageException;
+ boolean send(AuditEventBase[] logs) throws AuditMessageException;
- boolean sendStringified(String log) throws AuditMessageException;
+ boolean sendStringified(String log) throws AuditMessageException;
- boolean sendStringified(String[] logs) throws AuditMessageException;
+ boolean sendStringified(String[] logs) throws AuditMessageException;
- boolean flush();
+ boolean flush();
- /**
- * Name for the destination
- *
- * @return
- */
- String getName();
+ /**
+ * Name for the destination
+ *
+ * @return
+ */
+ String getName();
}
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
index 8004b75dd..ad11e5eac 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java
@@ -16,6 +16,22 @@
*/
package org.apache.ranger.audit.provider;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.Subject;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.login.LoginContext;
+
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
@@ -28,11 +44,13 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
@@ -44,971 +62,985 @@ import java.util.TimeZone;
import java.util.UUID;
import java.util.regex.Pattern;
-import javax.security.auth.Subject;
-import javax.security.auth.login.AppConfigurationEntry;
-import javax.security.auth.login.Configuration;
-import javax.security.auth.login.LoginContext;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.util.KerberosName;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
-import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonParser;
-
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
public class MiscUtil {
- private static final Logger logger = LoggerFactory.getLogger(MiscUtil.class);
-
- public static final String TOKEN_START = "%";
- public static final String TOKEN_END = "%";
- public static final String TOKEN_HOSTNAME = "hostname";
- public static final String TOKEN_APP_TYPE = "app-type";
- public static final String TOKEN_JVM_INSTANCE = "jvm-instance";
- public static final String TOKEN_TIME = "time:";
- public static final String TOKEN_PROPERTY = "property:";
- public static final String TOKEN_ENV = "env:";
- public static final String ESCAPE_STR = "\\";
-
- private static final VMID sJvmID = new VMID();
-
- public static String LINE_SEPARATOR = System.getProperty("line.separator");
-
- static private final ThreadLocal<ObjectMapper> MAPPER = new ThreadLocal<ObjectMapper>() {
- @Override
- protected ObjectMapper initialValue() {
- ObjectMapper objectMapper = new ObjectMapper();
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
- objectMapper.setDateFormat(dateFormat);
- objectMapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
- objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
- return objectMapper;
- }
- } ;
-
- static public ObjectMapper getMapper() {
- return MAPPER.get();
- }
- private static String sApplicationType = null;
- private static UserGroupInformation ugiLoginUser = null;
- private static Subject subjectLoginUser = null;
- private static String local_hostname = null;
-
- private static Map<String, LogHistory> logHistoryList = new Hashtable<String, LogHistory>();
- private static int logInterval = 30000; // 30 seconds
-
- static {
- initLocalHost();
- }
-
- public static String replaceTokens(String str, long time) {
- if (str == null) {
- return str;
- }
-
- if (time <= 0) {
- time = System.currentTimeMillis();
- }
-
- for (int startPos = 0; startPos < str.length();) {
- int tagStartPos = str.indexOf(TOKEN_START, startPos);
-
- if (tagStartPos == -1) {
- break;
- }
-
- int tagEndPos = str.indexOf(TOKEN_END,
- tagStartPos + TOKEN_START.length());
-
- if (tagEndPos == -1) {
- break;
- }
-
- String tag = str.substring(tagStartPos,
- tagEndPos + TOKEN_END.length());
- String token = tag.substring(TOKEN_START.length(),
- tag.lastIndexOf(TOKEN_END));
- String val = "";
-
- if (token != null) {
- if (token.equals(TOKEN_HOSTNAME)) {
- val = getHostname();
- } else if (token.equals(TOKEN_APP_TYPE)) {
- val = getApplicationType();
- } else if (token.equals(TOKEN_JVM_INSTANCE)) {
- val = getJvmInstanceId();
- } else if (token.startsWith(TOKEN_PROPERTY)) {
- String propertyName = token.substring(TOKEN_PROPERTY
- .length());
-
- val = getSystemProperty(propertyName);
- } else if (token.startsWith(TOKEN_ENV)) {
- String envName = token.substring(TOKEN_ENV.length());
-
- val = getEnv(envName);
- } else if (token.startsWith(TOKEN_TIME)) {
- String dtFormat = token.substring(TOKEN_TIME.length());
-
- val = getFormattedTime(time, dtFormat);
- }
- }
-
- if (val == null) {
- val = "";
- }
-
- str = str.substring(0, tagStartPos) + val
- + str.substring(tagEndPos + TOKEN_END.length());
- startPos = tagStartPos + val.length();
- }
-
- return str;
- }
-
- public static String getHostname() {
- String ret = local_hostname;
-
- if (ret == null) {
- initLocalHost();
-
- ret = local_hostname;
-
- if (ret == null) {
- ret = "unknown";
- }
- }
-
- return ret;
- }
-
- public static void setApplicationType(String applicationType) {
- sApplicationType = applicationType;
- }
-
- public static String getApplicationType() {
- return sApplicationType;
- }
-
- public static String getJvmInstanceId() {
- Integer val = Integer.valueOf(sJvmID.toString().hashCode());
- long longVal = val.longValue();
- String ret = Long.toString(Math.abs(longVal));
-
- return ret;
- }
-
- public static String getSystemProperty(String propertyName) {
- String ret = null;
-
- try {
- ret = propertyName != null ? System.getProperty(propertyName)
- : null;
- } catch (Exception excp) {
- logger.warn("getSystemProperty(" + propertyName + ") failed", excp);
- }
-
- return ret;
- }
-
- public static String getEnv(String envName) {
- String ret = null;
-
- try {
- ret = envName != null ? System.getenv(envName) : null;
- } catch (Exception excp) {
- logger.warn("getenv(" + envName + ") failed", excp);
- }
-
- return ret;
- }
-
- public static String getFormattedTime(long time, String format) {
- String ret = null;
-
- try {
- SimpleDateFormat sdf = new SimpleDateFormat(format);
-
- ret = sdf.format(time);
- } catch (Exception excp) {
- logger.warn("SimpleDateFormat.format() failed: " + format, excp);
- }
-
- return ret;
- }
-
- public static void createParents(File file) {
- if (file != null) {
- String parentName = file.getParent();
+ private static final Logger logger = LoggerFactory.getLogger(MiscUtil.class);
+
+ public static final String TOKEN_START = "%";
+ public static final String TOKEN_END = "%";
+ public static final String TOKEN_HOSTNAME = "hostname";
+ public static final String TOKEN_APP_TYPE = "app-type";
+ public static final String TOKEN_JVM_INSTANCE = "jvm-instance";
+ public static final String TOKEN_TIME = "time:";
+ public static final String TOKEN_PROPERTY = "property:";
+ public static final String TOKEN_ENV = "env:";
+ public static final String ESCAPE_STR = "\\";
+ public static final String LINE_SEPARATOR = System.lineSeparator();
+
+ private static String sApplicationType;
+ private static UserGroupInformation ugiLoginUser;
+ private static Subject subjectLoginUser;
+ private static String localHostname;
+ private static final Map<String, LogHistory> logHistoryList = new Hashtable<>();
+ private static final int logInterval = 30000; // 30 seconds
+ private static final VMID sJvmID = new VMID();
+ private static final ThreadLocal<ObjectMapper> MAPPER = ThreadLocal.withInitial(() -> {
+ ObjectMapper objectMapper = new ObjectMapper();
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+
+ objectMapper.setDateFormat(dateFormat);
+ objectMapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
+ objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
+
+ return objectMapper;
+ });
+
+ private MiscUtil() {
+ // to block instantiation
+ }
+
+ public static ObjectMapper getMapper() {
+ return MAPPER.get();
+ }
+
+ public static String replaceTokens(String str, long time) {
+ if (str == null) {
+ return str;
+ }
+
+ if (time <= 0) {
+ time = System.currentTimeMillis();
+ }
+
+ for (int startPos = 0; startPos < str.length(); ) {
+ int tagStartPos = str.indexOf(TOKEN_START, startPos);
- if (parentName != null) {
- File parentDir = new File(parentName);
+ if (tagStartPos == -1) {
+ break;
+ }
- if (!parentDir.exists()) {
- if (!parentDir.mkdirs()) {
- logger.warn("createParents(): failed to create "
- + parentDir.getAbsolutePath());
- }
- }
- }
- }
- }
+ int tagEndPos = str.indexOf(TOKEN_END, tagStartPos + TOKEN_START.length());
- public static long getNextRolloverTime(long lastRolloverTime, long interval) {
- long now = System.currentTimeMillis() / 1000 * 1000; // round to second
+ if (tagEndPos == -1) {
+ break;
+ }
- if (lastRolloverTime <= 0) {
- // should this be set to the next multiple-of-the-interval from
- // start of the day?
- return now + interval;
- } else if (lastRolloverTime <= now) {
- long nextRolloverTime = now + interval;
+ String tag = str.substring(tagStartPos, tagEndPos + TOKEN_END.length());
+ String token = tag.substring(TOKEN_START.length(), tag.lastIndexOf(TOKEN_END));
+ String val = "";
- // keep it at 'interval' boundary
- long trimInterval = (nextRolloverTime - lastRolloverTime)
- % interval;
+ if (token != null) {
+ if (token.equals(TOKEN_HOSTNAME)) {
+ val = getHostname();
+ } else if (token.equals(TOKEN_APP_TYPE)) {
+ val = getApplicationType();
+ } else if (token.equals(TOKEN_JVM_INSTANCE)) {
+ val = getJvmInstanceId();
+ } else if (token.startsWith(TOKEN_PROPERTY)) {
+ String propertyName = token.substring(TOKEN_PROPERTY.length());
- return nextRolloverTime - trimInterval;
- } else {
- return lastRolloverTime;
- }
- }
-
- public static long getRolloverStartTime(long nextRolloverTime, long interval) {
- return (nextRolloverTime <= interval) ? System.currentTimeMillis()
- : nextRolloverTime - interval;
- }
-
- public static int parseInteger(String str, int defValue) {
- int ret = defValue;
-
- if (str != null) {
- try {
- ret = Integer.parseInt(str);
- } catch (Exception excp) {
- // ignore
- }
- }
-
- return ret;
- }
-
- public static String generateUniqueId() {
- return UUID.randomUUID().toString();
- }
-
- // UUID.randomUUID() uses SecureRandom, which is seen to be slow in some environments; this method uses Random
- public static String generateGuid() {
- byte[] randomBytes = new byte[16];
-
- RandomHolder.random.nextBytes(randomBytes);
-
- UUID uuid = UUID.nameUUIDFromBytes(randomBytes);
-
- return uuid.toString();
- }
+ val = getSystemProperty(propertyName);
+ } else if (token.startsWith(TOKEN_ENV)) {
+ String envName = token.substring(TOKEN_ENV.length());
- public static <T> String stringify(T log) {
- String ret = null;
-
- if (log != null) {
- if (log instanceof String) {
- ret = (String) log;
+ val = getEnv(envName);
+ } else if (token.startsWith(TOKEN_TIME)) {
+ String dtFormat = token.substring(TOKEN_TIME.length());
+
+ val = getFormattedTime(time, dtFormat);
+ }
+ }
+
+ if (val == null) {
+ val = "";
+ }
+
+ str = str.substring(0, tagStartPos) + val + str.substring(tagEndPos + TOKEN_END.length());
+ startPos = tagStartPos + val.length();
+ }
+
+ return str;
+ }
+
+ public static String getHostname() {
+ String ret = localHostname;
+
+ if (ret == null) {
+ initLocalHost();
+
+ ret = localHostname;
+
+ if (ret == null) {
+ ret = "unknown";
+ }
+ }
+
+ return ret;
+ }
+
+ public static String getApplicationType() {
+ return sApplicationType;
+ }
+
+ public static void setApplicationType(String applicationType) {
+ sApplicationType = applicationType;
+ }
+
+ public static String getJvmInstanceId() {
+ int val = sJvmID.toString().hashCode();
+
+ return Long.toString(Math.abs((long) val));
+ }
+
+ public static String getSystemProperty(String propertyName) {
+ String ret = null;
+
+ try {
+ ret = propertyName != null ? System.getProperty(propertyName) : null;
+ } catch (Exception excp) {
+ logger.warn("getSystemProperty({}) failed", propertyName, excp);
+ }
+
+ return ret;
+ }
+
+ public static String getEnv(String envName) {
+ String ret = null;
+
+ try {
+ ret = envName != null ? System.getenv(envName) : null;
+ } catch (Exception excp) {
+ logger.warn("getenv({}) failed", envName, excp);
+ }
+
+ return ret;
+ }
+
+ public static String getFormattedTime(long time, String format) {
+ String ret = null;
+
+ try {
+ SimpleDateFormat sdf = new SimpleDateFormat(format);
+
+ ret = sdf.format(time);
+ } catch (Exception excp) {
+ logger.warn("SimpleDateFormat.format() failed: {}", format, excp);
+ }
+
+ return ret;
+ }
+
+ public static void createParents(File file) {
+ if (file != null) {
+ String parentName = file.getParent();
+
+ if (parentName != null) {
+ File parentDir = new File(parentName);
+
+ if (!parentDir.exists()) {
+ if (!parentDir.mkdirs()) {
+ logger.warn("createParents(): failed to create {}", parentDir.getAbsolutePath());
+ }
+ }
+ }
+ }
+ }
+
+ public static long getNextRolloverTime(long lastRolloverTime, long interval) {
+ long now = System.currentTimeMillis() / 1000 * 1000; // round to second
+
+ if (lastRolloverTime <= 0) {
+ // should this be set to the next multiple-of-the-interval from
+ // start of the day?
+ return now + interval;
+ } else if (lastRolloverTime <= now) {
+ long nextRolloverTime = now + interval;
+
+ // keep it at 'interval' boundary
+ long trimInterval = (nextRolloverTime - lastRolloverTime) % interval;
+
+ return nextRolloverTime - trimInterval;
+ } else {
+ return lastRolloverTime;
+ }
+ }
+
+ public static long getRolloverStartTime(long nextRolloverTime, long interval) {
+ return (nextRolloverTime <= interval) ? System.currentTimeMillis() : nextRolloverTime - interval;
+ }
+
+ public static int parseInteger(String str, int defValue) {
+ int ret = defValue;
+
+ if (str != null) {
+ try {
+ ret = Integer.parseInt(str);
+ } catch (Exception excp) {
+ // ignore
+ }
+ }
+
+ return ret;
+ }
+
+ public static String generateUniqueId() {
+ return UUID.randomUUID().toString();
+ }
+
+ // UUID.randomUUID() uses SecureRandom, which is seen to be slow in some environments; this method uses Random
+ public static String generateGuid() {
+ byte[] randomBytes = new byte[16];
+
+ RandomHolder.random.nextBytes(randomBytes);
+
+ UUID uuid = UUID.nameUUIDFromBytes(randomBytes);
+
+ return uuid.toString();
+ }
+
+ public static <T> String stringify(T log) {
+ String ret = null;
+
+ if (log != null) {
+ if (log instanceof String) {
+ ret = (String) log;
} else if (getMapper() != null) {
try {
ret = getMapper().writeValueAsString(log);
} catch (Exception e) {
- logger.error("Error occurred while processing JSOn object " + log, e);
+ logger.error("Error occurred while processing JSOn object {}", log, e);
+
ret = log.toString(); // Fallback to default toString() method
}
- } else {
- ret = log.toString();
- }
- }
+ } else {
+ ret = log.toString();
+ }
+ }
- return ret;
- }
+ return ret;
+ }
- static public <T> T fromJson(String jsonStr, Class<T> clazz) {
+ public static <T> T fromJson(String jsonStr, Class<T> clazz) {
try {
return getMapper().readValue(jsonStr, clazz);
} catch (Exception exception) {
- logger.error("Error occurred while processing JSOn object " + jsonStr, exception);
+ logger.error("Error occurred while processing JSOn object {}", jsonStr, exception);
}
+
return null;
- }
-
- public static String getStringProperty(Properties props, String propName) {
- String ret = null;
-
- if (props != null && propName != null) {
- String val = props.getProperty(propName);
- if (val != null) {
- ret = val;
- }
- }
-
- return ret;
- }
-
- public static String getStringProperty(Properties props, String propName, String defValue) {
- String ret = defValue;
-
- if (props != null && propName != null) {
- String val = props.getProperty(propName);
- if (val != null) {
- ret = val;
- }
- }
-
- return ret;
- }
-
- public static boolean getBooleanProperty(Properties props, String propName,
- boolean defValue) {
- boolean ret = defValue;
-
- if (props != null && propName != null) {
- String val = props.getProperty(propName);
-
- if (val != null) {
- ret = Boolean.valueOf(val);
- }
... 13478 lines suppressed ...