You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2018/09/26 15:42:49 UTC

[7/9] ranger git commit: RANGER-1501: Audit Flush to HDFS does not actually cause the audit logs to be flushed to HDFS - improvement patch

RANGER-1501: Audit Flush to HDFS does not actually cause the audit logs to be flushed to HDFS - improvement patch

Signed-off-by: rmani <rm...@hortonworks.com>
(cherry picked from commit 1befffcc1c18679cdf2c6413496b2078f6d2a838)


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/a3be2390
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/a3be2390
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/a3be2390

Branch: refs/heads/ranger-0.7
Commit: a3be2390eeae26fa41147270d358e51a552fa55c
Parents: 302c20a
Author: Vishal Suvagia <vi...@apache.org>
Authored: Wed Sep 26 19:41:08 2018 +0530
Committer: Pradeep <pr...@apache.org>
Committed: Wed Sep 26 20:55:33 2018 +0530

----------------------------------------------------------------------
 .../audit/destination/HDFSAuditDestination.java | 26 ++++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/a3be2390/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
index 889b6ff..8b17fc5 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
@@ -63,6 +63,7 @@ public class HDFSAuditDestination extends AuditDestination {
 	private String logFolder;
 
 	private PrintWriter logWriter = null;
+	volatile FSDataOutputStream ostream = null; // output stream wrapped in logWriter
 
 	private String currentFileName;
 
@@ -169,6 +170,7 @@ public class HDFSAuditDestination extends AuditDestination {
 				addDeferredCount(events.size());
 				out.close();
 				logWriter = null;
+				ostream = null;
 				return false;
 			}
 		} catch (Throwable t) {
@@ -187,10 +189,22 @@ public class HDFSAuditDestination extends AuditDestination {
 
 	@Override
 	public void flush() {
-		if ( logWriter != null) {
-			logWriter.flush();
-			logger.info("Flush HDFS audit logs completed.....");
-		 }
+		logger.info("Flush called. name=" + getName());
+		if (ostream != null) {
+			try {
+				synchronized (this) {
+					if (ostream != null)
+						// 1) PrinterWriter does not have bufferring of its own so
+						// we need to flush its underlying stream
+						// 2) HDFS flush() does not really flush all the way to disk.
+						ostream.hflush();
+						logger.info("Flush HDFS audit logs completed.....");
+				}
+			} catch (IOException e) {
+				logger.error("Error on flushing log writer: " + e.getMessage() +
+				 "\nException will be ignored. name=" + getName() + ", fileName=" + currentFileName);
+			}
+		}
 	}
 
 	/*
@@ -246,6 +260,7 @@ public class HDFSAuditDestination extends AuditDestination {
 						+ getName() + ", fileName=" + currentFileName);
 			}
 			logWriter = null;
+			ostream = null;
 		}
 		logStatus();
 	}
@@ -290,7 +305,7 @@ public class HDFSAuditDestination extends AuditDestination {
 
 			// Create the file to write
 			logger.info("Creating new log file. hdfPath=" + fullPath);
-			FSDataOutputStream ostream = fileSystem.create(hdfPath);
+			ostream = fileSystem.create(hdfPath);
 			logWriter = new PrintWriter(ostream);
 			currentFileName = fullPath;
 		}
@@ -341,6 +356,7 @@ public class HDFSAuditDestination extends AuditDestination {
 			}
 
 			logWriter = null;
+			ostream = null;
 			currentFileName = null;
 
 			if (!rollOverByDuration) {