You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2014/09/23 04:30:37 UTC
[1/2] git commit: ARGUS-5: fix to use logged user context to work
with HDFS file system (instead of the current user while the dispatcher
thread was created).
Repository: incubator-argus
Updated Branches:
refs/heads/master acd088613 -> 81f76076e
ARGUS-5: fix to use logged user context to work with HDFS file system
(instead of the current user while the dispatcher thread was created).
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/40ab8ebc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/40ab8ebc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/40ab8ebc
Branch: refs/heads/master
Commit: 40ab8ebcd69e5000d8fd44a306edd36d2138ec51
Parents: cfec85a
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 18:52:04 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 18:52:04 2014 -0700
----------------------------------------------------------------------
.../audit/provider/LocalFileLogBuffer.java | 94 +++++++++++++-------
.../com/xasecure/audit/provider/MiscUtil.java | 2 +-
.../audit/provider/hdfs/HdfsLogDestination.java | 15 ++--
3 files changed, 72 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/40ab8ebc/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 753c2e4..c6f2d6f 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -31,10 +31,12 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
+import java.security.PrivilegedAction;
import java.util.Arrays;
import java.util.Comparator;
import java.util.TreeSet;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.helpers.LogLog;
@@ -155,7 +157,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
public synchronized boolean add(T log) {
boolean ret = false;
- rolloverIfNeeded();
+ checkFileStatus();
Writer writer = mWriter;
@@ -183,11 +185,13 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
private synchronized void openFile() {
LogLog.debug("==> LocalFileLogBuffer.openFile()");
- long currentRolloverStartTime = MiscUtil.getCurrentRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
-
closeFile();
- mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, currentRolloverStartTime);
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+
+ long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+
+ mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
FileOutputStream ostream = null;
try {
@@ -206,8 +210,6 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
if(mWriter != null) {
LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
-
- mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
} else {
LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
@@ -250,11 +252,13 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
LogLog.debug("<== LocalFileLogBuffer.rollover()");
}
- private void rolloverIfNeeded() {
+ private void checkFileStatus() {
long now = System.currentTimeMillis();
if(now > mNextRolloverTime) {
rollover();
+ } else if(mWriter == null) {
+ openFile();
}
}
@@ -336,9 +340,33 @@ class DestinationDispatcherThread<T> extends Thread {
@Override
public void run() {
+ UserGroupInformation loginUser = null;
+
+ try {
+ loginUser = UserGroupInformation.getLoginUser();
+ } catch (IOException excp) {
+ LogLog.error("DestinationDispatcherThread.run(): failed to get login user details. Audit files will not be sent to HDFS destination", excp);
+ }
+
+ if(loginUser == null) {
+ LogLog.error("DestinationDispatcherThread.run(): failed to get login user. Audit files will not be sent to HDFS destination");
+
+ return;
+ }
+
+ loginUser.doAs(new PrivilegedAction<Integer>() {
+ @Override
+ public Integer run() {
+ doRun();
+
+ return 0;
+ }
+ });
+ }
+
+ private void doRun() {
init();
-
- // destination start() should be from the dispatcher thread
+
mDestination.start();
int pollIntervalInMs = 1000;
@@ -422,7 +450,7 @@ class DestinationDispatcherThread<T> extends Thread {
return ret;
}
-
+
private String getNextStringifiedLog() {
String log = null;
@@ -430,25 +458,29 @@ class DestinationDispatcherThread<T> extends Thread {
try {
while(true) {
String line = mReader.readLine();
-
- if(line == null) {
+
+ if(line == null) { // reached end-of-file
break;
- } else {
- if(log == null) {
- log = "";
- }
+ }
- if(line.endsWith(MiscUtil.ESCAPE_STR)) {
- line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
-
+ if(line.endsWith(MiscUtil.ESCAPE_STR)) {
+ line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
+
+ if(log == null) {
+ log = line;
+ } else {
log += MiscUtil.LINE_SEPARATOR;
log += line;
-
- continue;
+ }
+
+ continue;
+ } else {
+ if(log == null) {
+ log = line;
} else {
log += line;
- break;
}
+ break;
}
}
} catch (IOException excp) {
@@ -508,6 +540,7 @@ class DestinationDispatcherThread<T> extends Thread {
if(! logFile.renameTo(archiveFile)) {
// TODO: renameTo() does not work in all cases. in case of failure, copy the file contents to the destination and delete the file
+ LogLog.warn("archiving failed to move file: " + mCurrentLogfile + " ==> " + archiveFilename);
}
File archiveDir = new File(archiveDirName);
@@ -522,17 +555,16 @@ class DestinationDispatcherThread<T> extends Thread {
if(numOfFilesToDelete > 0) {
Arrays.sort(files, new Comparator<File>() {
-
- @Override
- public int compare(File f1, File f2) {
- return (int)(f1.lastModified() - f2.lastModified());
- }
- });
+ @Override
+ public int compare(File f1, File f2) {
+ return (int)(f1.lastModified() - f2.lastModified());
+ }
+ });
for(int i = 0; i < numOfFilesToDelete; i++) {
- LogLog.debug("DELETE: " + files[i].getAbsolutePath());
-
- files[i].delete();
+ if(! files[i].delete()) {
+ LogLog.warn("archiving failed to delete file: " + files[i].getAbsolutePath());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/40ab8ebc/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
index 3a07710..d382ead 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -136,7 +136,7 @@ public class MiscUtil {
}
}
- public static long getCurrentRolloverStartTime(long nextRolloverTime, long interval) {
+ public static long getRolloverStartTime(long nextRolloverTime, long interval) {
return (nextRolloverTime <= interval) ? System.currentTimeMillis() : nextRolloverTime - interval;
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/40ab8ebc/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index defb39f..0b6dd6e 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -136,7 +136,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
public boolean sendStringified(String log) {
boolean ret = false;
- checkDestinationFileStatus();
+ checkFileStatus();
OutputStreamWriter writer = mWriter;
@@ -158,11 +158,13 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
private void openFile() {
LogLog.debug("==> HdfsLogDestination.openFile()");
- long currentRolloverStartTime = MiscUtil.getCurrentRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
-
closeFile();
- mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, currentRolloverStartTime);
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+
+ long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+
+ mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
FSDataOutputStream ostream = null;
FileSystem fileSystem = null;
@@ -186,7 +188,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
ostream = fileSystem.append(pathLogfile);
} catch(IOException excp) {
// append may not be supported by the filesystem. rename existing file and create a new one
- String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_TIME_END, currentRolloverStartTime);
+ String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_TIME_END, startTime);
String movedFilename = appendToFilename(mHdfsFilename, fileSuffix);
Path movedFilePath = new Path(movedFilename);
@@ -222,7 +224,6 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
if(mWriter != null) {
LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);
- mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
mLastOpenFailedTime = 0;
} else {
LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);
@@ -265,7 +266,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
LogLog.debug("<== HdfsLogDestination.rollover()");
}
- private void checkDestinationFileStatus() {
+ private void checkFileStatus() {
long now = System.currentTimeMillis();
if(mWriter == null) {
[2/2] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-argus
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-argus
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/81f76076
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/81f76076
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/81f76076
Branch: refs/heads/master
Commit: 81f76076e4575d49bcffdd2447381b58af6be4f0
Parents: 40ab8eb acd0886
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 19:30:24 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 19:30:24 2014 -0700
----------------------------------------------------------------------
.../main/java/com/xasecure/biz/AssetMgr.java | 17 ++++++++++--
.../main/java/com/xasecure/biz/XABizUtil.java | 29 +++++++++++++-------
.../main/java/com/xasecure/rest/AssetREST.java | 2 ++
.../com/xasecure/service/XResourceService.java | 23 ++++++++++++++--
.../src/main/resources/xa_system.properties | 3 ++
5 files changed, 59 insertions(+), 15 deletions(-)
----------------------------------------------------------------------