You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2022/03/08 08:41:06 UTC
[ranger] branch ranger-2.3 updated: RANGER-3603:HDFS audit files rollover improvement to trigger rollover in monitoring thread
This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch ranger-2.3
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.3 by this push:
new f8be6c0 RANGER-3603:HDFS audit files rollover improvement to trigger rollover in monitoring thread
f8be6c0 is described below
commit f8be6c040bf6951708ec58232891334d177fc677
Author: Ramesh Mani <rm...@cloudera.com>
AuthorDate: Tue Mar 8 00:40:42 2022 -0800
RANGER-3603:HDFS audit files rollover improvement to trigger rollover in monitoring thread
---
.../audit/utils/AbstractRangerAuditWriter.java | 2 +-
.../ranger/audit/utils/RangerJSONAuditWriter.java | 81 ++++++++++++++++++++--
2 files changed, 78 insertions(+), 5 deletions(-)
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
index 191871c..ea35ff3 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/utils/AbstractRangerAuditWriter.java
@@ -216,7 +216,7 @@ public abstract class AbstractRangerAuditWriter implements RangerAuditWriter {
return;
}
- if ( System.currentTimeMillis() > nextRollOverTime.getTime() ) {
+ if ( System.currentTimeMillis() >= nextRollOverTime.getTime() ) {
logger.info("Closing file. Rolling over. name=" + auditProviderName
+ ", fileName=" + currentFileName);
try {
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
index 284a23d..7e8a431 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerJSONAuditWriter.java
@@ -26,6 +26,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.PrintWriter;
import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;
@@ -36,15 +40,44 @@ import java.util.Properties;
public class RangerJSONAuditWriter extends AbstractRangerAuditWriter {
private static final Logger logger = LoggerFactory.getLogger(RangerJSONAuditWriter.class);
+ public static final String PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER = "file.rollover.enable.periodic.rollover";
+ public static final String PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME = "file.rollover.periodic.rollover.check.sec";
protected String JSON_FILE_EXTENSION = ".log";
- public void init(Properties props, String propPrefix, String auditProviderName, Map<String,String> auditConfigs) {
+ /*
+ * When enableAuditFilePeriodicRollOver is enabled, Audit File in HDFS would be closed by the defined period in
+ * xasecure.audit.destination.hdfs.file.rollover.sec. By default xasecure.audit.destination.hdfs.file.rollover.sec = 86400 sec
+ * and file will be closed midnight. Custom rollover time can be set by defining file.rollover.sec to desire time in seconds.
+ */
+ private boolean enableAuditFilePeriodicRollOver = false;
+
+ /*
+ Time frequency of next occurrence of periodic rollover check. By Default every 60 seconds the check is done.
+ */
+ private long periodicRollOverCheckTimeinSec;
+
+ public void init(Properties props, String propPrefix, String auditProviderName, Map<String, String> auditConfigs) {
if (logger.isDebugEnabled()) {
logger.debug("==> RangerJSONAuditWriter.init()");
}
init();
- super.init(props,propPrefix,auditProviderName,auditConfigs);
+ super.init(props, propPrefix, auditProviderName, auditConfigs);
+
+ // start AuditFilePeriodicRollOverTask if enabled.
+ enableAuditFilePeriodicRollOver = MiscUtil.getBooleanProperty(props, propPrefix + "." + PROP_HDFS_ROLLOVER_ENABLE_PERIODIC_ROLLOVER, false);
+ if (enableAuditFilePeriodicRollOver) {
+ periodicRollOverCheckTimeinSec = MiscUtil.getLongProperty(props, propPrefix + "." + PROP_HDFS_ROLLOVER_PERIODIC_ROLLOVER_CHECK_TIME, 60L);
+ try {
+ if (logger.isDebugEnabled()) {
+ logger.debug("rolloverPeriod: " + rolloverPeriod + " nextRollOverTime: " + nextRollOverTime + " periodicRollOverTimeinSec: " + periodicRollOverCheckTimeinSec);
+ }
+ startAuditFilePeriodicRollOverTask();
+ } catch (Exception e) {
+ logger.warn("Error enabling audit file perodic rollover..! Default behavior will be");
+ }
+ }
+
if (logger.isDebugEnabled()) {
logger.debug("<== RangerJSONAuditWriter.init()");
}
@@ -128,7 +161,11 @@ public class RangerJSONAuditWriter extends AbstractRangerAuditWriter {
}
synchronized public PrintWriter getLogFileStream() throws Exception {
- closeFileIfNeeded();
+ if (!enableAuditFilePeriodicRollOver) {
+ // when periodic rollover is enabled closing of file is done by the file rollover monitoring task and hence don't need to
+ // close the file inline with audit logging.
+ closeFileIfNeeded();
+ }
// Either there are no open log file or the previous one has been rolled
// over
PrintWriter logWriter = createWriter();
@@ -172,4 +209,40 @@ public class RangerJSONAuditWriter extends AbstractRangerAuditWriter {
logger.debug("<== JSONWriter.stop()");
}
}
-}
+
+ private void startAuditFilePeriodicRollOverTask() {
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new AuditFilePeriodicRollOverTaskThreadFactory());
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("HDFSAuditDestination.startAuditFilePeriodicRollOverTask() strated.." + "Audit File rollover happens every " + rolloverPeriod );
+ }
+
+ executorService.scheduleAtFixedRate(new AuditFilePeriodicRollOverTask(), 0, periodicRollOverCheckTimeinSec, TimeUnit.SECONDS);
+ }
+
+ class AuditFilePeriodicRollOverTaskThreadFactory implements ThreadFactory {
+ //Threadfactory to create a daemon Thread.
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r, "AuditFilePeriodicRollOverTask");
+ t.setDaemon(true);
+ return t;
+ }
+ }
+
+ private class AuditFilePeriodicRollOverTask implements Runnable {
+ public void run() {
+ if (logger.isDebugEnabled()) {
+ logger.debug("==> AuditFilePeriodicRollOverTask.run()");
+ }
+ try {
+ closeFileIfNeeded();
+ } catch (Exception excp) {
+ logger.error("AuditFilePeriodicRollOverTask Failed. Aborting..", excp);
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("<== AuditFilePeriodicRollOverTask.run()");
+ }
+ }
+ }
+
+}
\ No newline at end of file