You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2014/09/23 20:33:28 UTC
[1/5] git commit: ARGUS-5: added periodic flush to local and HDFS
files;
and configurable parameters for the same in xasecure-audit.xml and
install.properties
Repository: incubator-argus
Updated Branches:
refs/heads/master 6c11ec265 -> 647cdb7ee
ARGUS-5: added periodic flush to local and HDFS files; and configurable
parameters for the same in xasecure-audit.xml and install.properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/74d83939
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/74d83939
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/74d83939
Branch: refs/heads/master
Commit: 74d83939558f6378ccb2cc2e4e6498cc980b1776
Parents: 81f7607
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 21:02:23 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 21:02:23 2014 -0700
----------------------------------------------------------------------
.../audit/provider/LocalFileLogBuffer.java | 23 +++++++++++++++++++-
.../audit/provider/hdfs/HdfsAuditProvider.java | 4 ++++
.../audit/provider/hdfs/HdfsLogDestination.java | 20 +++++++++++++++++
hbase-agent/conf/xasecure-audit-changes.cfg | 2 ++
hbase-agent/conf/xasecure-audit.xml | 10 +++++++++
hbase-agent/scripts/install.properties | 2 ++
hdfs-agent/conf/xasecure-audit-changes.cfg | 2 ++
hdfs-agent/conf/xasecure-audit.xml | 10 +++++++++
hdfs-agent/scripts/install.properties | 2 ++
hive-agent/conf/xasecure-audit-changes.cfg | 2 ++
hive-agent/conf/xasecure-audit.xml | 10 +++++++++
hive-agent/scripts/install.properties | 2 ++
knox-agent/conf/xasecure-audit-changes.cfg | 2 ++
knox-agent/conf/xasecure-audit.xml | 10 +++++++++
knox-agent/scripts/install.properties | 2 ++
storm-agent/conf/xasecure-audit-changes.cfg | 2 ++
storm-agent/conf/xasecure-audit.xml | 10 +++++++++
storm-agent/scripts/install.properties | 2 ++
18 files changed, 116 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index c6f2d6f..40d9c20 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -34,6 +34,7 @@ import java.io.Writer;
import java.security.PrivilegedAction;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.Date;
import java.util.TreeSet;
import org.apache.hadoop.security.UserGroupInformation;
@@ -43,15 +44,17 @@ import org.apache.log4j.helpers.LogLog;
public class LocalFileLogBuffer<T> implements LogBuffer<T> {
private String mDirectory = null;
private String mFile = null;
+ private int mFlushIntervalSeconds = 1 * 60;
private String mEncoding = null;
private boolean mIsAppend = true;
- private int mRolloverIntervalSeconds = 600;
+ private int mRolloverIntervalSeconds = 10 * 60;
private String mArchiveDirectory = null;
private int mArchiveFileCount = 10;
private Writer mWriter = null;
private String mBufferFilename = null;
private long mNextRolloverTime = 0;
+ private long mNextFlushTime = 0;
private DestinationDispatcherThread<T> mDispatcherThread = null;
@@ -74,6 +77,14 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
mFile = file;
}
+ public int getFlushIntervalSeconds() {
+ return mFlushIntervalSeconds;
+ }
+
+ public void setFlushIntervalSeconds(int flushIntervalSeconds) {
+ mFlushIntervalSeconds = flushIntervalSeconds;
+ }
+
public String getEncoding() {
return mEncoding;
}
@@ -210,6 +221,8 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
if(mWriter != null) {
LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
+
+ mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000);
} else {
LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
@@ -259,6 +272,14 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
rollover();
} else if(mWriter == null) {
openFile();
+ } else if(now > mNextFlushTime) {
+ try {
+ mWriter.flush();
+
+ mNextFlushTime = now + (mFlushIntervalSeconds * 1000);
+ } catch (IOException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to flush", excp);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
index e8b3922..ffe84c5 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -17,11 +17,13 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
String hdfsDestinationDirectory = properties.get("destination.directroy");
String hdfsDestinationFile = properties.get("destination.file");
+ int hdfsDestinationFlushIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.flush.interval.seconds"), 15 * 60);
int hdfsDestinationRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.rollover.interval.seconds"), 24 * 60 * 60);
int hdfsDestinationOpenRetryIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.open.retry.interval.seconds"), 60);
String localFileBufferDirectory = properties.get("local.buffer.directroy");
String localFileBufferFile = properties.get("local.buffer.file");
+ int localFileBufferFlushIntervalSeconds = MiscUtil.parseInteger(properties.get("local.buffer.flush.interval.seconds"), 1 * 60);
int localFileBufferRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("local.buffer.rollover.interval.seconds"), 10 * 60);
String localFileBufferArchiveDirectory = properties.get("local.archive.directroy");
int localFileBufferArchiveFileCount = MiscUtil.parseInteger(properties.get("local.archive.max.file.count"), 10);
@@ -30,6 +32,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
mHdfsDestination.setDirectory(hdfsDestinationDirectory);
mHdfsDestination.setFile(hdfsDestinationFile);
+ mHdfsDestination.setFlushIntervalSeconds(hdfsDestinationFlushIntervalSeconds);
mHdfsDestination.setEncoding(encoding);
mHdfsDestination.setRolloverIntervalSeconds(hdfsDestinationRolloverIntervalSeconds);
mHdfsDestination.setOpenRetryIntervalSeconds(hdfsDestinationOpenRetryIntervalSeconds);
@@ -38,6 +41,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
mLocalFileBuffer.setDirectory(localFileBufferDirectory);
mLocalFileBuffer.setFile(localFileBufferFile);
+ mLocalFileBuffer.setFlushIntervalSeconds(localFileBufferFlushIntervalSeconds);
mLocalFileBuffer.setEncoding(encoding);
mLocalFileBuffer.setRolloverIntervalSeconds(localFileBufferRolloverIntervalSeconds);
mLocalFileBuffer.setArchiveDirectory(localFileBufferArchiveDirectory);
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index 0b6dd6e..42119f8 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -25,6 +25,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URI;
+import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -38,6 +39,7 @@ import com.xasecure.audit.provider.MiscUtil;
public class HdfsLogDestination<T> implements LogDestination<T> {
private String mDirectory = null;
private String mFile = null;
+ private int mFlushIntervalSeconds = 1 * 60;
private String mEncoding = null;
private boolean mIsAppend = true;
private int mRolloverIntervalSeconds = 24 * 60 * 60;
@@ -46,6 +48,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
private OutputStreamWriter mWriter = null;
private String mHdfsFilename = null;
private long mNextRolloverTime = 0;
+ private long mNextFlushTime = 0;
private long mLastOpenFailedTime = 0;
private boolean mIsStopInProgress = false;
@@ -68,6 +71,14 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
this.mFile = file;
}
+ public int getFlushIntervalSeconds() {
+ return mFlushIntervalSeconds;
+ }
+
+ public void setFlushIntervalSeconds(int flushIntervalSeconds) {
+ mFlushIntervalSeconds = flushIntervalSeconds;
+ }
+
public String getEncoding() {
return mEncoding;
}
@@ -224,6 +235,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
if(mWriter != null) {
LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);
+ mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000);
mLastOpenFailedTime = 0;
} else {
LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);
@@ -275,6 +287,14 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
}
} else if(now > mNextRolloverTime) {
rollover();
+ } else if(now > mNextFlushTime) {
+ try {
+ mWriter.flush();
+
+ mNextFlushTime = now + (mFlushIntervalSeconds * 1000);
+ } catch (IOException excp) {
+ LogLog.warn("HdfsLogDestination: failed to flush", excp);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit-changes.cfg b/hbase-agent/conf/xasecure-audit-changes.cfg
index 66beb58..48a8cf5 100644
--- a/hbase-agent/conf/xasecure-audit-changes.cfg
+++ b/hbase-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER%
xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit.xml b/hbase-agent/conf/xasecure-audit.xml
index 93f06d1..c57d580 100644
--- a/hbase-agent/conf/xasecure-audit.xml
+++ b/hbase-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+ <value>900</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
<value>86400</value>
</property>
@@ -151,6 +156,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
<value>600</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index 337b3f5..2e227dc 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -114,10 +114,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
XAAUDIT.HDFS.IS_ENABLED=false
XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hbase/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hbase
XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hbase
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit-changes.cfg b/hdfs-agent/conf/xasecure-audit-changes.cfg
index 66beb58..48a8cf5 100644
--- a/hdfs-agent/conf/xasecure-audit-changes.cfg
+++ b/hdfs-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER%
xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit.xml b/hdfs-agent/conf/xasecure-audit.xml
index e2b8f9d..e7c23d4 100644
--- a/hdfs-agent/conf/xasecure-audit.xml
+++ b/hdfs-agent/conf/xasecure-audit.xml
@@ -112,6 +112,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+ <value>900</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
<value>86400</value>
</property>
@@ -132,6 +137,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
<value>600</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 5cfa476..d8b6680 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -107,10 +107,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
XAAUDIT.HDFS.IS_ENABLED=false
XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hdfs/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hdfs
XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hdfs
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit-changes.cfg b/hive-agent/conf/xasecure-audit-changes.cfg
index 0ae94fb..e8547c2 100644
--- a/hive-agent/conf/xasecure-audit-changes.cfg
+++ b/hive-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER%
xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit.xml b/hive-agent/conf/xasecure-audit.xml
index 319ae09..35c4c8c 100644
--- a/hive-agent/conf/xasecure-audit.xml
+++ b/hive-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+ <value>900</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
<value>86400</value>
</property>
@@ -151,6 +156,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
<value>600</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 4cbdde0..1c4a232 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -114,10 +114,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
XAAUDIT.HDFS.IS_ENABLED=false
XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hive/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hive
XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hive
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit-changes.cfg b/knox-agent/conf/xasecure-audit-changes.cfg
index 9e0340e..02e6fb1 100644
--- a/knox-agent/conf/xasecure-audit-changes.cfg
+++ b/knox-agent/conf/xasecure-audit-changes.cfg
@@ -8,10 +8,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER%
xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit.xml b/knox-agent/conf/xasecure-audit.xml
index 4a7303e..c861abb 100644
--- a/knox-agent/conf/xasecure-audit.xml
+++ b/knox-agent/conf/xasecure-audit.xml
@@ -126,6 +126,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+ <value>900</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
<value>86400</value>
</property>
@@ -146,6 +151,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
<value>600</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index 1f0e01e..c93e844 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -100,10 +100,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
XAAUDIT.HDFS.IS_ENABLED=false
XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/knox/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/knox
XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/knox
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit-changes.cfg b/storm-agent/conf/xasecure-audit-changes.cfg
index 68a2484..bfbc90f 100644
--- a/storm-agent/conf/xasecure-audit-changes.cfg
+++ b/storm-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER%
xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit.xml b/storm-agent/conf/xasecure-audit.xml
index f8c07d2..acac19b 100644
--- a/storm-agent/conf/xasecure-audit.xml
+++ b/storm-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+ <value>900</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
<value>86400</value>
</property>
@@ -151,6 +156,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
<value>600</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index c1e3e44..2d28d5d 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -104,10 +104,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
XAAUDIT.HDFS.IS_ENABLED=false
XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/storm/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/storm
XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/storm
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
[2/5] git commit: ARGUS-5: dev-test update for HDFS audit provider.
Posted by ma...@apache.org.
ARGUS-5: dev-test update for HDFS audit provider.
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/ea4df438
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/ea4df438
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/ea4df438
Branch: refs/heads/master
Commit: ea4df438291cb981326056adb3eb3e1bf349c66f
Parents: 74d8393
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 23:56:40 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 23:56:40 2014 -0700
----------------------------------------------------------------------
.../audit/provider/BufferedAuditProvider.java | 8 ++++++
.../audit/provider/LocalFileLogBuffer.java | 14 ++++++++--
.../com/xasecure/audit/provider/LogBuffer.java | 2 ++
.../audit/provider/hdfs/HdfsAuditProvider.java | 17 +++++++++++-
.../com/xasecure/audit/test/TestEvents.java | 4 +++
.../src/scripts/xasecure-audit.properties | 29 ++++++++++++++++----
6 files changed, 65 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
index 9b8cb40..3db5f91 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
@@ -57,6 +57,14 @@ public abstract class BufferedAuditProvider implements AuditProvider {
public void flush() {
}
+ protected LogBuffer<AuditEventBase> getBuffer() {
+ return mBuffer;
+ }
+
+ protected LogDestination<AuditEventBase> getDestination() {
+ return mDestination;
+ }
+
protected void setBufferAndDestination(LogBuffer<AuditEventBase> buffer,
LogDestination<AuditEventBase> destination) {
mBuffer = buffer;
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 40d9c20..8738bf0 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -34,7 +34,6 @@ import java.io.Writer;
import java.security.PrivilegedAction;
import java.util.Arrays;
import java.util.Comparator;
-import java.util.Date;
import java.util.TreeSet;
import org.apache.hadoop.security.UserGroupInformation;
@@ -193,6 +192,11 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
return ret;
}
+ @Override
+ public boolean isEmpty() {
+ return mDispatcherThread == null || mDispatcherThread.isIdle();
+ }
+
private synchronized void openFile() {
LogLog.debug("==> LocalFileLogBuffer.openFile()");
@@ -283,7 +287,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
}
}
- public OutputStreamWriter createWriter(OutputStream os ) {
+ private OutputStreamWriter createWriter(OutputStream os ) {
OutputStreamWriter writer = null;
if(os != null) {
@@ -359,6 +363,12 @@ class DestinationDispatcherThread<T> extends Thread {
mStopThread = true;
}
+ public boolean isIdle() {
+ synchronized(mCompletedLogfiles) {
+ return mCompletedLogfiles.isEmpty() && mCurrentLogfile == null;
+ }
+ }
+
@Override
public void run() {
UserGroupInformation loginUser = null;
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
index ce67e01..d8ff10a 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
@@ -26,5 +26,7 @@ public interface LogBuffer<T> {
boolean isAvailable();
+ public boolean isEmpty();
+
public boolean add(T log);
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
index ffe84c5..6ab78ea 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -13,7 +13,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
}
public void init(Map<String, String> properties) {
- String encoding = properties.get("encoding");
+ String encoding = properties.get("encoding");
String hdfsDestinationDirectory = properties.get("destination.directroy");
String hdfsDestinationFile = properties.get("destination.file");
@@ -50,5 +50,20 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
setBufferAndDestination(mLocalFileBuffer, mHdfsDestination);
}
+ @Override
+ public void waitToComplete() {
+ while(getBuffer() != null && !getBuffer().isEmpty()) {
+ sleep(1000);
+ }
+ }
+
+ private static void sleep(int timeInMs) {
+ try {
+ Thread.sleep(timeInMs);
+ } catch(InterruptedException excp) {
+
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java b/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
index aca18fd..dd7e035 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
@@ -96,6 +96,10 @@ public class TestEvents {
if(i != 0 && ((i % 100) == 0))
Thread.sleep(100);
}
+
+ provider.waitToComplete();
+
+ provider.stop();
} catch(Exception excp) {
LOG.info(excp.getLocalizedMessage());
excp.printStackTrace();
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/lookup-client/src/scripts/xasecure-audit.properties
----------------------------------------------------------------------
diff --git a/lookup-client/src/scripts/xasecure-audit.properties b/lookup-client/src/scripts/xasecure-audit.properties
index 9989c00..904b19c 100644
--- a/lookup-client/src/scripts/xasecure-audit.properties
+++ b/lookup-client/src/scripts/xasecure-audit.properties
@@ -1,16 +1,33 @@
-xasecure.audit.jpa.javax.persistence.jdbc.url=jdbc:mysql://localhost:3306/xasecure
-xasecure.audit.jpa.javax.persistence.jdbc.user=xalogger
-xasecure.audit.jpa.javax.persistence.jdbc.password=xalogger
-xasecure.audit.jpa.javax.persistence.jdbc.driver=com.mysql.jdbc.Driver
-
xasecure.audit.is.enabled=true
+
+
xasecure.audit.log4j.is.enabled=false
xasecure.audit.log4j.is.async=false
xasecure.audit.log4j.async.max.queue.size=100000
xasecure.audit.log4j.async.max.flush.interval.ms=30000
-xasecure.audit.db.is.enabled=true
+
+
+xasecure.audit.db.is.enabled=false
xasecure.audit.db.is.async=true
xasecure.audit.db.async.max.queue.size=102400
xasecure.audit.db.async.resume.queue.size=92400
xasecure.audit.db.async.max.flush.interval.ms=30000
xasecure.audit.db.batch.size=100
+xasecure.audit.jpa.javax.persistence.jdbc.url=jdbc:mysql://localhost:3306/xasecure
+xasecure.audit.jpa.javax.persistence.jdbc.user=xalogger
+xasecure.audit.jpa.javax.persistence.jdbc.password=xalogger
+xasecure.audit.jpa.javax.persistence.jdbc.driver=com.mysql.jdbc.Driver
+
+
+xasecure.audit.hdfs.is.enabled=false
+xasecure.audit.hdfs.config.destination.directroy=hdfs://%hostname%:8020/tmp/audit/hdfs/%time:yyyyMMdd%
+xasecure.audit.hdfs.config.destination.file=%hostname%-audit.log
+xasecure.audit.hdfs.config.destination.flush.interval.seconds=900
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds=86400
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds=60
+xasecure.audit.hdfs.config.local.buffer.directroy=/tmp/audit/hdfs
+xasecure.audit.hdfs.config.local.buffer.file=%time:yyyyMMdd-HHmm.ss%.log
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds=60
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds=600
+xasecure.audit.hdfs.config.local.archive.directroy=/tmp/audit/archive/hdfs
+xasecure.audit.hdfs.config.local.archive.max.file.count=10
[3/5] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-argus
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-argus
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/7201bf87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/7201bf87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/7201bf87
Branch: refs/heads/master
Commit: 7201bf87c8001cfd8657a298faf2809b0a0a5028
Parents: ea4df43 6c11ec2
Author: mneethiraj <mn...@hortonworks.com>
Authored: Tue Sep 23 10:00:42 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Tue Sep 23 10:00:42 2014 -0700
----------------------------------------------------------------------
security-admin/scripts/install.sh | 4 ++--
security-admin/src/main/webapp/login.jsp | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[5/5] git commit: ARGUS-5: fix per review comments
Posted by ma...@apache.org.
ARGUS-5: fix per review comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/647cdb7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/647cdb7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/647cdb7e
Branch: refs/heads/master
Commit: 647cdb7eea386b34db1c300996a13b848eefc361
Parents: b6efd47
Author: mneethiraj <mn...@hortonworks.com>
Authored: Tue Sep 23 11:32:49 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Tue Sep 23 11:32:49 2014 -0700
----------------------------------------------------------------------
.../xasecure/audit/provider/LocalFileLogBuffer.java | 14 +++++++-------
.../audit/provider/hdfs/HdfsLogDestination.java | 10 +++++-----
2 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/647cdb7e/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 8738bf0..767cbc4 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -202,9 +202,9 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
closeFile();
- mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
- long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
@@ -226,7 +226,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
if(mWriter != null) {
LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
- mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000);
+ mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L);
} else {
LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
@@ -280,7 +280,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
try {
mWriter.flush();
- mNextFlushTime = now + (mFlushIntervalSeconds * 1000);
+ mNextFlushTime = now + (mFlushIntervalSeconds * 1000L);
} catch (IOException excp) {
LogLog.warn("LocalFileLogBuffer: failed to flush", excp);
}
@@ -400,7 +400,7 @@ class DestinationDispatcherThread<T> extends Thread {
mDestination.start();
- int pollIntervalInMs = 1000;
+ long pollIntervalInMs = 1000L;
while(! mStopThread) {
synchronized(mCompletedLogfiles) {
@@ -452,7 +452,7 @@ class DestinationDispatcherThread<T> extends Thread {
boolean ret = false;
- int destinationPollIntervalInMs = 1000;
+ long destinationPollIntervalInMs = 1000L;
openCurrentFile();
@@ -628,7 +628,7 @@ class DestinationDispatcherThread<T> extends Thread {
return reader;
}
- private void sleep(int sleepTimeInMs, String onFailMsg) {
+ private void sleep(long sleepTimeInMs, String onFailMsg) {
try {
Thread.sleep(sleepTimeInMs);
} catch(InterruptedException excp) {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/647cdb7e/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index 42119f8..c524e0b 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -171,9 +171,9 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
closeFile();
- mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
- long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));
mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
@@ -235,7 +235,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
if(mWriter != null) {
LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);
- mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000);
+ mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L);
mLastOpenFailedTime = 0;
} else {
LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);
@@ -282,7 +282,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
long now = System.currentTimeMillis();
if(mWriter == null) {
- if(now > (mLastOpenFailedTime + (mOpenRetryIntervalSeconds * 1000))) {
+ if(now > (mLastOpenFailedTime + (mOpenRetryIntervalSeconds * 1000L))) {
openFile();
}
} else if(now > mNextRolloverTime) {
@@ -291,7 +291,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
try {
mWriter.flush();
- mNextFlushTime = now + (mFlushIntervalSeconds * 1000);
+ mNextFlushTime = now + (mFlushIntervalSeconds * 1000L);
} catch (IOException excp) {
LogLog.warn("HdfsLogDestination: failed to flush", excp);
}
[4/5] git commit: ARGUS-5: audit test code/config updated to enable
testing with HDFS audit store.
Posted by ma...@apache.org.
ARGUS-5: audit test code/config updated to enable testing with HDFS
audit store.
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/b6efd471
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/b6efd471
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/b6efd471
Branch: refs/heads/master
Commit: b6efd47136bf449ce12966af9cd2e9b398cf6415
Parents: 7201bf8
Author: mneethiraj <mn...@hortonworks.com>
Authored: Tue Sep 23 11:12:35 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Tue Sep 23 11:12:35 2014 -0700
----------------------------------------------------------------------
.../audit/provider/hdfs/HdfsAuditProvider.java | 17 -------------
.../com/xasecure/audit/test/TestEvents.java | 26 ++++++++++++++++----
lookup-client/src/scripts/log4j.xml | 1 +
lookup-client/src/scripts/run-audit-test.sh | 9 +++++--
.../src/scripts/xasecure-audit.properties | 5 ++++
5 files changed, 34 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/b6efd471/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
index 6ab78ea..3f3132c 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -49,21 +49,4 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
setBufferAndDestination(mLocalFileBuffer, mHdfsDestination);
}
-
- @Override
- public void waitToComplete() {
- while(getBuffer() != null && !getBuffer().isEmpty()) {
- sleep(1000);
- }
- }
-
- private static void sleep(int timeInMs) {
- try {
- Thread.sleep(timeInMs);
- } catch(InterruptedException excp) {
-
- }
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/b6efd471/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java b/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
index dd7e035..8968093 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
@@ -28,7 +28,6 @@ import java.util.Date;
import java.util.Properties;
import com.xasecure.audit.model.AuditEventBase;
-import com.xasecure.audit.model.EnumRepositoryType;
import com.xasecure.audit.model.HBaseAuditEvent;
import com.xasecure.audit.model.HdfsAuditEvent;
import com.xasecure.audit.model.HiveAuditEvent;
@@ -83,9 +82,13 @@ public class TestEvents {
LOG.info("provider=" + provider.toString());
- String strEventCount = args.length > 0 ? args[0] : auditProperties.getProperty("xasecure.audit.test.event.count");
+ String strEventCount = args.length > 0 ? args[0] : auditProperties.getProperty("xasecure.audit.test.event.count");
+ String strEventPauseTimeInMs = args.length > 1 ? args[1] : auditProperties.getProperty("xasecure.audit.test.event.pause.time.ms");
+ String strSleepTimeBeforeExit = args.length > 2 ? args[2] : auditProperties.getProperty("xasecure.audit.test.sleep.time.before.exit.seconds");
- int eventCount = (strEventCount == null) ? 1024 : Integer.parseInt(strEventCount);
+ int eventCount = (strEventCount == null) ? 1024 : Integer.parseInt(strEventCount);
+ int eventPauseTime = (strEventPauseTimeInMs == null) ? 0 : Integer.parseInt(strEventPauseTimeInMs);
+ int sleepTimeBeforeExit = ((strSleepTimeBeforeExit == null) ? 0 : Integer.parseInt(strSleepTimeBeforeExit)) * 1000;
for(int i = 0; i < eventCount; i++) {
AuditEventBase event = getTestEvent(i);
@@ -93,11 +96,24 @@ public class TestEvents {
LOG.info("==> TestEvents.main(" + (i+1) + "): adding " + event.getClass().getName());
provider.log(event);
- if(i != 0 && ((i % 100) == 0))
- Thread.sleep(100);
+ if(eventPauseTime > 0) {
+ Thread.sleep(eventPauseTime);
+ }
}
provider.waitToComplete();
+
+ // incase of HdfsAuditProvider, logs are saved to local file system which gets sent to HDFS asynchronusly in a separate thread.
+ // So, at this point it is possible that few local log files haven't made to HDFS.
+ if(sleepTimeBeforeExit > 0) {
+ LOG.info("waiting for " + sleepTimeBeforeExit + "ms before exiting..");
+
+ try {
+ Thread.sleep(sleepTimeBeforeExit);
+ } catch(Exception excp) {
+ LOG.info("error while waiting before exiting..");
+ }
+ }
provider.stop();
} catch(Exception excp) {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/b6efd471/lookup-client/src/scripts/log4j.xml
----------------------------------------------------------------------
diff --git a/lookup-client/src/scripts/log4j.xml b/lookup-client/src/scripts/log4j.xml
index 20bb2db..f31717d 100644
--- a/lookup-client/src/scripts/log4j.xml
+++ b/lookup-client/src/scripts/log4j.xml
@@ -20,6 +20,7 @@
<root>
<priority value ="info" />
+ <appender-ref ref="console" />
<appender-ref ref="logFile" />
</root>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/b6efd471/lookup-client/src/scripts/run-audit-test.sh
----------------------------------------------------------------------
diff --git a/lookup-client/src/scripts/run-audit-test.sh b/lookup-client/src/scripts/run-audit-test.sh
index d1a944f..693aa5c 100755
--- a/lookup-client/src/scripts/run-audit-test.sh
+++ b/lookup-client/src/scripts/run-audit-test.sh
@@ -1,9 +1,14 @@
#!/bin/bash
-HADOOP_DIR=/usr/lib/hadoop
+HADOOP_DIR=/usr/hdp/current/hadoop
HADOOP_CONF_DIR=/etc/hadoop/conf
-cp="$HADOOP_CONF_DIR:$HADOOP_DIR/lib/*:$HADOOP_DIR/client/hadoop-common.jar"
+cp=
+for jar in $HADOOP_CONF_DIR $HADOOP_DIR/lib/* $HADOOP_DIR/client/*
+do
+ cp=${cp}:${jar}
+done
+
export cp
java -Xmx1024M -Xms1024M -cp "${cp}" com.xasecure.audit.test.TestEvents $*
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/b6efd471/lookup-client/src/scripts/xasecure-audit.properties
----------------------------------------------------------------------
diff --git a/lookup-client/src/scripts/xasecure-audit.properties b/lookup-client/src/scripts/xasecure-audit.properties
index 904b19c..2b755f1 100644
--- a/lookup-client/src/scripts/xasecure-audit.properties
+++ b/lookup-client/src/scripts/xasecure-audit.properties
@@ -1,3 +1,8 @@
+xasecure.audit.test.event.count=1000
+xasecure.audit.test.event.pause.time.ms=100
+xasecure.audit.test.sleep.time.before.exit.seconds=60
+
+
xasecure.audit.is.enabled=true