You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2014/09/22 20:29:15 UTC
[01/10] git commit: Log4j appender to write to HDFS.
Repository: incubator-argus
Updated Branches:
refs/heads/master 1e1fcff18 -> cfec85a65
Log4j appender to write to HDFS.
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/be47e640
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/be47e640
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/be47e640
Branch: refs/heads/master
Commit: be47e6402804b013cfcf76aae27f899010b721ed
Parents: daac1c6
Author: mneethiraj <mn...@hortonworks.com>
Authored: Thu Sep 18 23:40:57 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Thu Sep 18 23:40:57 2014 -0700
----------------------------------------------------------------------
.../audit/provider/Log4jAuditProvider.java | 2 +-
.../audit/provider/MultiDestAuditProvider.java | 38 +-
agents-common/conf/log4j.properties | 18 +
.../java/org/apache/log4j/BufferedAppender.java | 123 ++++
.../org/apache/log4j/HdfsLogDestination.java | 297 ++++++++++
.../apache/log4j/HdfsRollingFileAppender.java | 130 ++++
.../org/apache/log4j/LocalFileLogBuffer.java | 591 +++++++++++++++++++
.../main/java/org/apache/log4j/LogBuffer.java | 30 +
.../java/org/apache/log4j/LogDestination.java | 30 +
9 files changed, 1245 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-audit/src/main/java/com/xasecure/audit/provider/Log4jAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/Log4jAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/Log4jAuditProvider.java
index e9c7bd4..dc6a27f 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/Log4jAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/Log4jAuditProvider.java
@@ -27,7 +27,7 @@ import com.xasecure.audit.model.AuditEventBase;
public class Log4jAuditProvider implements AuditProvider {
private static final Log LOG = LogFactory.getLog(Log4jAuditProvider.class);
- private static final Log AUDITLOG = LogFactory.getLog("xaaudit." + Log4jAuditProvider.class);
+ private static final Log AUDITLOG = LogFactory.getLog("xaaudit." + Log4jAuditProvider.class.getName());
public Log4jAuditProvider() {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-audit/src/main/java/com/xasecure/audit/provider/MultiDestAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MultiDestAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MultiDestAuditProvider.java
index d068bdd..cee9f5a 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MultiDestAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MultiDestAuditProvider.java
@@ -67,33 +67,45 @@ public class MultiDestAuditProvider implements AuditProvider {
@Override
public void log(AuditEventBase event) {
- try {
- for(AuditProvider provider : mProviders) {
+ for(AuditProvider provider : mProviders) {
+ try {
provider.log(event);
- }
- } catch(Throwable excp) {
- LOG.error("failed to log event { " + event.toString() + " }", excp);
- }
+ } catch(Throwable excp) {
+ LOG.error("AsyncAuditProvider.log(): failed for provider { " + provider.getClass().getName() + " }", excp);
+ }
+ }
}
@Override
public void start() {
for(AuditProvider provider : mProviders) {
- provider.start();
+ try {
+ provider.start();
+ } catch(Throwable excp) {
+ LOG.error("AsyncAuditProvider.start(): failed for provider { " + provider.getClass().getName() + " }", excp);
+ }
}
}
@Override
public void stop() {
for(AuditProvider provider : mProviders) {
- provider.stop();
+ try {
+ provider.stop();
+ } catch(Throwable excp) {
+ LOG.error("AsyncAuditProvider.stop(): failed for provider { " + provider.getClass().getName() + " }", excp);
+ }
}
}
@Override
public void waitToComplete() {
for(AuditProvider provider : mProviders) {
- provider.waitToComplete();
+ try {
+ provider.waitToComplete();
+ } catch(Throwable excp) {
+ LOG.error("AsyncAuditProvider.waitToComplete(): failed for provider { " + provider.getClass().getName() + " }", excp);
+ }
}
}
@@ -126,12 +138,12 @@ public class MultiDestAuditProvider implements AuditProvider {
@Override
public void flush() {
- try {
- for(AuditProvider provider : mProviders) {
+ for(AuditProvider provider : mProviders) {
+ try {
provider.flush();
+ } catch(Throwable excp) {
+ LOG.error("AsyncAuditProvider.flush(): failed for provider { " + provider.getClass().getName() + " }", excp);
}
- } catch(Throwable excp) {
- LOG.error("AsyncAuditProvider.flush(): failed to flush events", excp);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/agents-common/conf/log4j.properties b/agents-common/conf/log4j.properties
new file mode 100644
index 0000000..c1ea266
--- /dev/null
+++ b/agents-common/conf/log4j.properties
@@ -0,0 +1,18 @@
+log4j.logger.xaaudit.com.xasecure.audit.provider.Log4jAuditProvider=INFO, hdfsAppender
+
+
+log4j.appender.hdfsAppender=org.apache.log4j.HdfsRollingFileAppender
+log4j.appender.hdfsAppender.hdfsDestinationDirectory=hdfs://%hostname%:8020/logs/application/%file-open-time:yyyyMMdd%
+log4j.appender.hdfsAppender.hdfsDestinationFile=%hostname%-audit.log
+log4j.appender.hdfsAppender.hdfsDestinationRolloverIntervalSeconds=86400
+
+log4j.appender.hdfsAppender.localFileBufferDirectory=/tmp/logs/application/%hostname%
+log4j.appender.hdfsAppender.localFileBufferFile=%file-open-time:yyyyMMdd-HHmm.ss%.log
+log4j.appender.hdfsAppender.localFileBufferRolloverIntervalSeconds=15
+log4j.appender.hdfsAppender.localFileBufferArchiveDirectory=/tmp/logs/archive/application/%hostname%
+log4j.appender.hdfsAppender.localFileBufferArchiveFileCount=12
+
+
+log4j.appender.hdfsAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.hdfsAppender.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
+log4j.appender.hdfsAppender.encoding=UTF-8
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/src/main/java/org/apache/log4j/BufferedAppender.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/BufferedAppender.java b/agents-common/src/main/java/org/apache/log4j/BufferedAppender.java
new file mode 100644
index 0000000..916d2eb
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/log4j/BufferedAppender.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.log4j;
+
+
+import org.apache.log4j.helpers.LogLog;
+
+public abstract class BufferedAppender<T> extends AppenderSkeleton {
+ private LogDestination<T> mLogDestination = null;
+ private LogBuffer<T> mLogBuffer = null;
+
+ protected void setBufferAndDestination(LogBuffer<T> buffer, LogDestination<T> destination) {
+ close();
+
+ mLogBuffer = buffer;
+ mLogDestination = destination;
+ }
+
+ protected boolean isLogable() {
+ return mLogBuffer != null;
+ }
+
+ protected void addToBuffer(T log) {
+ if(mLogBuffer != null) {
+ mLogBuffer.add(log);
+ }
+ }
+
+ protected void start() {
+ LogLog.debug("==> BufferedAppender.start()");
+
+ if(mLogBuffer == null) {
+ LogLog.warn("BufferedAppender.start(): logBuffer is null");
+ }
+
+ if(mLogDestination == null) {
+ LogLog.warn("BufferedAppender.start(): logDestination is null");
+ }
+
+ if(mLogBuffer != null && mLogDestination != null) {
+ JVMShutdownHook jvmShutdownHook = new JVMShutdownHook(this);
+
+ Runtime.getRuntime().addShutdownHook(jvmShutdownHook);
+
+ mLogBuffer.start(mLogDestination);
+ }
+
+ LogLog.debug("<== BufferedAppender.start()");
+ }
+
+ protected void stop() {
+ LogLog.debug("==> BufferedAppender.stop()");
+
+ LogBuffer<T> tmpBuff = mLogBuffer;
+
+ mLogDestination = null;
+ mLogBuffer = null;
+
+ if(tmpBuff != null) {
+ tmpBuff.stop();
+ }
+
+ LogLog.debug("<== BufferedAppender.stop()");
+ }
+
+ @Override
+ public void close() {
+ LogLog.debug("==> BufferedAppender.close()");
+
+ stop();
+
+ LogLog.debug("<== BufferedAppender.close()");
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("BufferedAppender {");
+ if(mLogDestination != null) {
+ sb.append(mLogDestination.toString());
+ }
+
+ sb.append(" ");
+
+ if(mLogBuffer != null) {
+ sb.append(mLogBuffer.toString());
+ }
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+ private class JVMShutdownHook extends Thread {
+ Appender mAppender = null;
+
+ public JVMShutdownHook(Appender appender) {
+ mAppender = appender;
+ }
+
+ public void run() {
+ if(mAppender != null) {
+ mAppender.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java b/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
new file mode 100644
index 0000000..ec53977
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.log4j;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.LocalFileLogBuffer.MiscUtil;
+import org.apache.log4j.helpers.LogLog;
+
+public class HdfsLogDestination implements LogDestination<String> {
+ private String mDirectory = null;
+ private String mFile = null;
+ private String mEncoding = null;
+ private boolean mIsAppend = true;
+ private int mRolloverIntervalSeconds = 24 * 60 * 60;
+
+ private OutputStreamWriter mWriter = null;
+ private String mCurrentFilename = null;
+ private long mNextRolloverTime = 0;
+
+ private String LINE_SEPARATOR = System.getProperty("line.separator");
+
+ public HdfsLogDestination() {
+ }
+
+ public String getDirectory() {
+ return mDirectory;
+ }
+
+ public void setDirectory(String directory) {
+ this.mDirectory = directory;
+ }
+
+ public String getFile() {
+ return mFile;
+ }
+
+ public void setFile(String file) {
+ this.mFile = file;
+ }
+
+ public String getEncoding() {
+ return mEncoding;
+ }
+
+ public void setEncoding(String encoding) {
+ mEncoding = encoding;
+ }
+
+ public int getRolloverIntervalSeconds() {
+ return mRolloverIntervalSeconds;
+ }
+
+ public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
+ this.mRolloverIntervalSeconds = rolloverIntervalSeconds;
+ }
+
+ @Override
+ public void start() {
+ LogLog.debug("==> HdfsLogDestination.start()");
+
+ openFile();
+
+ LogLog.debug("<== HdfsLogDestination.start()");
+ }
+
+ @Override
+ public void stop() {
+ LogLog.debug("==> HdfsLogDestination.stop()");
+
+ closeFile();
+
+ LogLog.debug("<== HdfsLogDestination.stop()");
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return mWriter != null;
+ }
+
+ @Override
+ public boolean add(String log) {
+ boolean ret = false;
+
+ long now = System.currentTimeMillis();
+
+ if(now > mNextRolloverTime) {
+ rollover();
+ }
+
+ OutputStreamWriter writer = mWriter;
+
+ if(writer != null) {
+ try {
+ writer.write(log + LINE_SEPARATOR);
+ writer.flush();
+
+ ret = true;
+ } catch (IOException excp) {
+ LogLog.warn("HdfsLogDestination.add(): write failed", excp);
+ }
+ }
+
+ return ret;
+ }
+
+ private void openFile() {
+ LogLog.debug("==> HdfsLogDestination.openFile()");
+
+ closeFile();
+
+ mCurrentFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+
+ FSDataOutputStream ostream = null;
+ FileSystem fileSystem = null;
+ Path pathLogfile = null;
+ Configuration conf = null;
+
+ try {
+ LogLog.debug("HdfsLogDestination.openFile(): opening file " + mCurrentFilename);
+
+ URI uri = URI.create(mCurrentFilename);
+
+ // TODO: mechanism to notify co-located XA-HDFS plugin to disable auditing of access checks to the current HDFS file
+ // this can be driven by adding an option (property) the logger, which can be configured at the deployment time.
+ // Like: hdfsCurrentFilenameProperty. When this option is set, do the following here:
+ // System.setProperty(hdfsCurrentFilenameProperty, uri.getPath());
+
+ conf = new Configuration();
+ pathLogfile = new Path(mCurrentFilename);
+ fileSystem = FileSystem.get(uri, conf);
+
+ if(fileSystem.exists(pathLogfile)) {
+ if(mIsAppend) {
+ try {
+ ostream = fileSystem.append(pathLogfile);
+ } catch(IOException excp) {
+ // append may not be supported by the filesystem. rename existing file and create a new one
+ String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_FILE_OPEN_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_FILE_OPEN_TIME_END);
+ String movedFilename = appendToFilename(mCurrentFilename, fileSuffix);
+ Path movedFilePath = new Path(movedFilename);
+
+ fileSystem.rename(pathLogfile, movedFilePath);
+ }
+ }
+ }
+
+ if(ostream == null){
+ ostream = fileSystem.create(pathLogfile);
+ }
+ } catch(IOException ex) {
+ Path parentPath = pathLogfile.getParent();
+
+ try {
+ if(parentPath != null&& fileSystem != null && !fileSystem.exists(parentPath) && fileSystem.mkdirs(parentPath)) {
+ ostream = fileSystem.create(pathLogfile);
+ }
+ } catch (IOException e) {
+ LogLog.warn("HdfsLogDestination.openFile() failed", e);
+ } catch (Throwable e) {
+ LogLog.warn("HdfsLogDestination.openFile() failed", e);
+ }
+ } catch(Throwable ex) {
+ LogLog.warn("HdfsLogDestination.openFile() failed", ex);
+ } finally {
+ // TODO: unset the property set above to exclude auditing of logfile opening
+ // System.setProperty(hdfsCurrentFilenameProperty, null);
+ }
+
+ mWriter = createWriter(ostream);
+
+ if(mWriter != null) {
+ LogLog.debug("HdfsLogDestination.openFile(): opened file " + mCurrentFilename);
+
+ updateNextRolloverTime();
+ } else {
+ LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mCurrentFilename);
+
+ mCurrentFilename = null;
+ }
+
+ LogLog.debug("<== HdfsLogDestination.openFile(" + mCurrentFilename + ")");
+ }
+
+ private void closeFile() {
+ LogLog.debug("==> HdfsLogDestination.closeFile()");
+
+ OutputStreamWriter writer = mWriter;
+
+ mWriter = null;
+
+ if(writer != null) {
+ try {
+ writer.close();
+ } catch(IOException excp) {
+ LogLog.warn("HdfsLogDestination: failed to close file " + mCurrentFilename, excp);
+ }
+ }
+
+ LogLog.debug("<== HdfsLogDestination.closeFile()");
+ }
+
+ private void rollover() {
+ LogLog.debug("==> HdfsLogDestination.rollover()");
+
+ closeFile();
+
+ openFile();
+
+ LogLog.debug("<== HdfsLogDestination.rollover()");
+ }
+
+ private OutputStreamWriter createWriter(OutputStream os ) {
+ OutputStreamWriter writer = null;
+
+ if(os != null) {
+ if(mEncoding != null) {
+ try {
+ writer = new OutputStreamWriter(os, mEncoding);
+ } catch(UnsupportedEncodingException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to create output writer.", excp);
+ }
+ }
+
+ if(writer == null) {
+ writer = new OutputStreamWriter(os);
+ }
+ }
+
+ return writer;
+ }
+
+ private void updateNextRolloverTime() {
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ }
+
+ private String appendToFilename(String fileName, String strToAppend) {
+ String ret = fileName;
+
+ if(strToAppend != null) {
+ if(ret == null) {
+ ret = "";
+ }
+
+ int extnPos = ret.lastIndexOf(".");
+
+ if(extnPos < 0) {
+ ret += strToAppend;
+ } else {
+ String extn = ret.substring(extnPos);
+
+ ret = ret.substring(0, extnPos) + strToAppend + extn;
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("HdfsLogDestination {");
+ sb.append("Directory=").append(mDirectory).append("; ");
+ sb.append("File=").append(mFile).append("; ");
+ sb.append("RolloverIntervalSeconds=").append(mRolloverIntervalSeconds);
+ sb.append("}");
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/src/main/java/org/apache/log4j/HdfsRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/HdfsRollingFileAppender.java b/agents-common/src/main/java/org/apache/log4j/HdfsRollingFileAppender.java
new file mode 100644
index 0000000..a855575
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/log4j/HdfsRollingFileAppender.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.log4j;
+
+
+import org.apache.log4j.helpers.LogLog;
+import org.apache.log4j.spi.LoggingEvent;
+
+
+public class HdfsRollingFileAppender extends BufferedAppender<String> {
+ String mEncoding = null;
+
+ String mHdfsDestinationDirectory = null;
+ String mHdfsDestinationFile = null;
+ int mHdfsDestinationRolloverIntervalSeconds = 24 * 60 * 60;
+
+ String mLocalFileBufferDirectory = null;
+ String mLocalFileBufferFile = null;
+ int mLocalFileBufferRolloverIntervalSeconds = 10 * 60;
+ String mLocalFileBufferArchiveDirectory = null;
+ int mLocalFileBufferArchiveFileCount = 10;
+
+ public void setEncoding(String encoding) {
+ mEncoding = encoding;
+ }
+
+ public void setHdfsDestinationDirectory(String hdfsDestinationDirectory) {
+ mHdfsDestinationDirectory = hdfsDestinationDirectory;
+ }
+
+ public void setHdfsDestinationFile(String hdfsDestinationFile) {
+ mHdfsDestinationFile = hdfsDestinationFile;
+ }
+
+ public void setHdfsDestinationRolloverIntervalSeconds(int hdfsDestinationRolloverIntervalSeconds) {
+ mHdfsDestinationRolloverIntervalSeconds = hdfsDestinationRolloverIntervalSeconds;
+ }
+
+ public void setLocalFileBufferDirectory(String localFileBufferDirectory) {
+ mLocalFileBufferDirectory = localFileBufferDirectory;
+ }
+
+ public void setLocalFileBufferFile(String localFileBufferFile) {
+ mLocalFileBufferFile = localFileBufferFile;
+ }
+
+ public void setLocalFileBufferRolloverIntervalSeconds(int localFileBufferRolloverIntervalSeconds) {
+ mLocalFileBufferRolloverIntervalSeconds = localFileBufferRolloverIntervalSeconds;
+ }
+
+ public void setLocalFileBufferArchiveDirectory(String localFileBufferArchiveDirectory) {
+ mLocalFileBufferArchiveDirectory = localFileBufferArchiveDirectory;
+ }
+
+ public void setLocalFileBufferArchiveFileCount(int localFileBufferArchiveFileCount) {
+ mLocalFileBufferArchiveFileCount = localFileBufferArchiveFileCount;
+ }
+
+ @Override
+ public boolean requiresLayout() {
+ return true;
+ }
+
+ @Override
+ public void activateOptions() {
+ LogLog.debug("==> HdfsRollingFileAppender.activateOptions()");
+
+ HdfsLogDestination hdfsDestination = new HdfsLogDestination();
+
+ hdfsDestination.setDirectory(mHdfsDestinationDirectory);
+ hdfsDestination.setFile(mHdfsDestinationFile);
+ hdfsDestination.setEncoding(mEncoding);
+ hdfsDestination.setRolloverIntervalSeconds(mHdfsDestinationRolloverIntervalSeconds);
+
+ LocalFileLogBuffer localFileBuffer = new LocalFileLogBuffer();
+
+ localFileBuffer.setDirectory(mLocalFileBufferDirectory);
+ localFileBuffer.setFile(mLocalFileBufferFile);
+ localFileBuffer.setEncoding(mEncoding);
+ localFileBuffer.setRolloverIntervalSeconds(mLocalFileBufferRolloverIntervalSeconds);
+ localFileBuffer.setArchiveDirectory(mLocalFileBufferArchiveDirectory);
+ localFileBuffer.setArchiveFileCount(mLocalFileBufferArchiveFileCount);
+
+ setBufferAndDestination(localFileBuffer, hdfsDestination);
+
+ start();
+
+ LogLog.debug("<== HdfsRollingFileAppender.activateOptions()");
+ }
+
+ @Override
+ protected void append(LoggingEvent event) {
+ if(isLogable()) {
+ String logMsg = this.layout.format(event);
+
+ if(layout.ignoresThrowable()) {
+ String[] strThrowable = event.getThrowableStrRep();
+ if (strThrowable != null) {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(logMsg);
+
+ for(String s : strThrowable) {
+ sb.append(s).append(Layout.LINE_SEP);
+ }
+
+ logMsg = sb.toString();
+ }
+ }
+
+ addToBuffer(logMsg);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java b/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
new file mode 100644
index 0000000..f52f042
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.log4j;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.rmi.dgc.VMID;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TreeSet;
+
+import org.apache.log4j.LocalFileLogBuffer.MiscUtil;
+import org.apache.log4j.helpers.LogLog;
+
+
+public class LocalFileLogBuffer implements LogBuffer<String> {
+ private static final int DEFAULT_ROLLOVER_INTERVAL = 600;
+
+ private String mDirectory = null;
+ private String mFile = null;
+ private String mEncoding = null;
+ private boolean mIsAppend = true;
+ private int mRolloverIntervalSeconds = DEFAULT_ROLLOVER_INTERVAL;
+ private String mArchiveDirectory = null;
+ private int mArchiveFileCount = 10;
+
+ private Writer mWriter = null;
+ private String mCurrentFilename = null;
+ private long mNextRolloverTime = 0;
+
+ private DestinationDispatcherThread mDispatcherThread = null;
+
+ public String getDirectory() {
+ return mDirectory;
+ }
+
+ public void setDirectory(String directory) {
+ mDirectory = directory;
+ }
+
+ public String getFile() {
+ return mFile;
+ }
+
+ public void setFile(String file) {
+ mFile = file;
+ }
+
+ public String getEncoding() {
+ return mEncoding;
+ }
+
+ public void setEncoding(String encoding) {
+ mEncoding = encoding;
+ }
+
+ public boolean getIsAppend() {
+ return mIsAppend;
+ }
+
+ public void setIsAppend(boolean isAppend) {
+ mIsAppend = isAppend;
+ }
+
+ public int getRolloverIntervalSeconds() {
+ return mRolloverIntervalSeconds;
+ }
+
+ public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
+ mRolloverIntervalSeconds = rolloverIntervalSeconds;
+ }
+
+ public String getArchiveDirectory() {
+ return mArchiveDirectory;
+ }
+
+ public void setArchiveDirectory(String archiveDirectory) {
+ mArchiveDirectory = archiveDirectory;
+ }
+
+ public int getArchiveFileCount() {
+ return mArchiveFileCount;
+ }
+
+ public void setArchiveFileCount(int archiveFileCount) {
+ mArchiveFileCount = archiveFileCount;
+ }
+
+
+ @Override
+ public void start(LogDestination<String> destination) {
+ LogLog.debug("==> LocalFileLogBuffer.start()");
+
+ openFile();
+
+ mDispatcherThread = new DestinationDispatcherThread(this, destination);
+
+ mDispatcherThread.start();
+
+ LogLog.debug("<== LocalFileLogBuffer.start()");
+ }
+
+ @Override
+ public void stop() {
+ LogLog.debug("==> LocalFileLogBuffer.stop()");
+
+ DestinationDispatcherThread dispatcherThread = mDispatcherThread;
+ mDispatcherThread = null;
+
+ if(dispatcherThread != null && dispatcherThread.isAlive()) {
+ dispatcherThread.stopThread();
+
+ try {
+ dispatcherThread.join();
+ } catch (InterruptedException e) {
+ LogLog.warn("LocalFileLogBuffer.stop(): failed in waiting for DispatcherThread", e);
+ }
+ }
+
+ closeFile();
+
+ LogLog.debug("<== LocalFileLogBuffer.stop()");
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return mWriter != null;
+ }
+
+ @Override
+ public boolean add(String log) {
+ boolean ret = false;
+
+ long now = System.currentTimeMillis();
+
+ if(now > mNextRolloverTime) {
+ rollover();
+ }
+
+ Writer writer = mWriter;
+
+ if(writer != null) {
+ try {
+ writer.write(log);
+
+ ret = true;
+ } catch(IOException excp) {
+ LogLog.warn("LocalFileLogBuffer.add(): write failed", excp);
+ }
+ } else {
+ LogLog.warn("LocalFileLogBuffer.add(): writer is null");
+ }
+
+ return ret;
+ }
+
+ private void openFile() {
+ LogLog.debug("==> LocalFileLogBuffer.openFile()");
+
+ closeFile();
+
+ mCurrentFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+
+ FileOutputStream ostream = null;
+ try {
+ ostream = new FileOutputStream(mCurrentFilename, mIsAppend);
+ } catch(Exception excp) {
+ MiscUtil.createParents(new File(mCurrentFilename));
+
+ try {
+ ostream = new FileOutputStream(mCurrentFilename, mIsAppend);
+ } catch(Exception ex) {
+ // ignore; error printed down
+ }
+ }
+
+ mWriter = createWriter(ostream);
+
+ if(mWriter != null) {
+ LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mCurrentFilename);
+
+ updateNextRolloverTime();
+ } else {
+ LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write" + mCurrentFilename);
+
+ mCurrentFilename = null;
+ }
+
+ LogLog.debug("<== LocalFileLogBuffer.openFile()");
+ }
+
+ private void closeFile() {
+ LogLog.debug("==> LocalFileLogBuffer.closeFile()");
+
+ Writer writer = mWriter;
+
+ mWriter = null;
+
+ if(writer != null) {
+ try {
+ writer.close();
+ } catch(IOException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to close file " + mCurrentFilename, excp);
+ }
+
+ if(mDispatcherThread != null) {
+ mDispatcherThread.addLogfile(mCurrentFilename);
+ }
+ }
+
+ LogLog.debug("<== LocalFileLogBuffer.closeFile()");
+ }
+
+ private void rollover() {
+ LogLog.debug("==> LocalFileLogBuffer.rollover()");
+
+ closeFile();
+
+ openFile();
+
+ LogLog.debug("<== LocalFileLogBuffer.rollover()");
+ }
+
+ public OutputStreamWriter createWriter(OutputStream os ) {
+ OutputStreamWriter writer = null;
+
+ if(os != null) {
+ if(mEncoding != null) {
+ try {
+ writer = new OutputStreamWriter(os, mEncoding);
+ } catch(UnsupportedEncodingException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to create output writer.", excp);
+ }
+ }
+
+ if(writer == null) {
+ writer = new OutputStreamWriter(os);
+ }
+ }
+
+ return writer;
+ }
+
+ private void updateNextRolloverTime() {
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ }
+
+ boolean isCurrentFilename(String filename) {
+ return mCurrentFilename != null && filename != null && filename.equals(mCurrentFilename);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("LocalFileLogBuffer {");
+ sb.append("Directory=").append(mDirectory).append("; ");
+ sb.append("File=").append(mFile).append("; ");
+ sb.append("RolloverIntervaSeconds=").append(mRolloverIntervalSeconds).append("; ");
+ sb.append("ArchiveDirectory=").append(mArchiveDirectory).append("; ");
+ sb.append("ArchiveFileCount=").append(mArchiveFileCount);
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+ static class MiscUtil {
+ static String TOKEN_HOSTNAME = "%hostname%";
+ static String TOKEN_APP_INSTANCE = "%app-instance%";
+ static String TOKEN_FILE_OPEN_TIME_START = "%file-open-time:";
+ static String TOKEN_FILE_OPEN_TIME_END = "%";
+
+ static VMID sJvmID = new VMID();
+
+ public static String replaceTokens(String str) {
+ if(str == null) {
+ return str;
+ }
+
+ str = replaceHostname(str);
+ str = replaceAppInstance(str);
+ str = replaceFileOpenTime(str);
+
+ return str;
+ }
+
+ public static String replaceHostname(String str) {
+ if(!str.contains(TOKEN_HOSTNAME)) {
+ return str;
+ }
+
+ String hostName = null;
+
+ try {
+ hostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException excp) {
+ LogLog.warn("LocalFileLogBuffer", excp);
+ }
+
+ if(hostName == null) {
+ hostName = "Unknown";
+ }
+
+ return str.replaceAll(TOKEN_HOSTNAME, hostName);
+ }
+
+ public static String replaceAppInstance(String str) {
+ if(!str.contains(TOKEN_APP_INSTANCE)) {
+ return str;
+ }
+
+ String appInstance = Integer.toString(Math.abs(sJvmID.hashCode()));
+
+ return str.replaceAll(TOKEN_APP_INSTANCE, appInstance);
+ }
+
+ public static String replaceFileOpenTime(String str) {
+ Date now = new Date();
+
+ while(str.contains(TOKEN_FILE_OPEN_TIME_START)) {
+ int tagStartPos = str.indexOf(TOKEN_FILE_OPEN_TIME_START);
+ int tagEndPos = str.indexOf(TOKEN_FILE_OPEN_TIME_END, tagStartPos + TOKEN_FILE_OPEN_TIME_START.length());
+
+ if(tagEndPos <= tagStartPos) {
+ break;
+ }
+
+ String tag = str.substring(tagStartPos, tagEndPos+1);
+ String dtFormat = tag.substring(TOKEN_FILE_OPEN_TIME_START.length(), tag.lastIndexOf(TOKEN_FILE_OPEN_TIME_END));
+
+ String replaceStr = "";
+
+ if(dtFormat != null) {
+ SimpleDateFormat sdf = new SimpleDateFormat(dtFormat);
+
+ replaceStr = sdf.format(now);
+ }
+
+ str = str.replaceAll(tag, replaceStr);
+ }
+
+ return str;
+ }
+
+ public static void createParents(File file) {
+ if(file != null) {
+ String parentName = file.getParent();
+
+ if (parentName != null) {
+ File parentDir = new File(parentName);
+
+ if(!parentDir.exists()) {
+ parentDir.mkdirs();
+ }
+ }
+ }
+ }
+
+ public static long getNextRolloverTime(long lastRolloverTime, long interval) {
+ long now = System.currentTimeMillis() / 1000 * 1000; // round to second
+
+ if(lastRolloverTime <= 0) {
+ // should this be set to the next multiple-of-the-interval from start of the day?
+ return now + interval;
+ } else if(lastRolloverTime <= now) {
+ long nextRolloverTime = now + interval;
+
+ // keep it at 'interval' boundary
+ long trimInterval = (nextRolloverTime - lastRolloverTime) % interval;
+
+ return nextRolloverTime - trimInterval;
+ } else {
+ return lastRolloverTime;
+ }
+ }
+ }
+}
+
+class DestinationDispatcherThread extends Thread {
+ private TreeSet<String> mCompletedLogfiles = new TreeSet<String>();
+ private boolean mStopThread = false;
+ private LocalFileLogBuffer mFileLogBuffer = null;
+ private LogDestination<String> mDestination = null;
+
+ private String mCurrentLogfile = null;
+ private BufferedReader mReader = null;
+
+ public DestinationDispatcherThread(LocalFileLogBuffer fileLogBuffer, LogDestination<String> destination) {
+ super(DestinationDispatcherThread.class.getSimpleName());
+
+ mFileLogBuffer = fileLogBuffer;
+ mDestination = destination;
+
+ setDaemon(true);
+ }
+
+ public void addLogfile(String filename) {
+ LogLog.debug("==> DestinationDispatcherThread.addLogfile(" + filename + ")");
+
+ synchronized(mCompletedLogfiles) {
+ mCompletedLogfiles.add(filename);
+ mCompletedLogfiles.notify();
+ }
+
+ LogLog.debug("<== DestinationDispatcherThread.addLogfile(" + filename + ")");
+ }
+
+ public String getNextLogfile() {
+ synchronized(mCompletedLogfiles) {
+ return mCompletedLogfiles.pollFirst();
+ }
+ }
+
+ public void stopThread() {
+ mStopThread = true;
+ }
+
+ @Override
+ public void run() {
+ init();
+
+ // destination start() should be from the dispatcher thread
+ mDestination.start();
+
+ int pollIntervalInMs = 1000;
+
+ while(! mStopThread) {
+ String logMsg = getNextLog();
+
+ if(logMsg == null) { // move to the next file
+ synchronized(mCompletedLogfiles) {
+ while(mCompletedLogfiles.isEmpty() && !mStopThread) {
+ try {
+ mCompletedLogfiles.wait(pollIntervalInMs);
+ } catch(InterruptedException excp) {
+ LogLog.warn("LocalFileLogBuffer.run(): failed to wait for log file", excp);
+ }
+ }
+
+ if(!mCompletedLogfiles.isEmpty()) {
+ openNextFile();
+ }
+ }
+ } else { // deliver to the msg to destination
+ while(! mDestination.add(logMsg) && !mStopThread) {
+ try {
+ Thread.sleep(pollIntervalInMs);
+ } catch(InterruptedException excp) {
+ LogLog.warn("LocalFileLogBuffer.run(): failed to wait for destination to be available", excp);
+ }
+ }
+ }
+ }
+
+ mDestination.stop();
+ }
+
+ private void init() {
+ LogLog.debug("==> DestinationDispatcherThread.init()");
+
+ String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory());
+ File directory = new File(dirName);
+
+ if(directory.exists() && directory.isDirectory()) {
+ File[] files = directory.listFiles();
+
+ if(files != null) {
+ for(File file : files) {
+ if(file.exists() && file.canRead()) {
+ String filename = file.getAbsolutePath();
+ if(! mFileLogBuffer.isCurrentFilename(filename)) {
+ addLogfile(filename);
+ }
+ }
+ }
+ }
+ }
+
+ openNextFile();
+
+ LogLog.debug("<== DestinationDispatcherThread.init()");
+ }
+
+ private String getNextLog() {
+ String log = null;
+
+ if(mReader != null) {
+ try {
+ log = mReader.readLine();
+ } catch (IOException excp) {
+ LogLog.warn("LocalFileLogBuffer.getNextLog(): failed to read from file " + mCurrentLogfile, excp);
+ }
+
+ if(log == null) {
+ closeCurrentFile();
+ }
+ }
+
+ return log;
+ }
+
+ private void openNextFile() {
+ LogLog.debug("==> openNextFile()");
+
+ closeCurrentFile();
+
+ while(mReader == null) {
+ mCurrentLogfile = getNextLogfile();
+
+ if(mCurrentLogfile != null) {
+ try {
+ FileReader fr = new FileReader(mCurrentLogfile);
+
+ mReader = new BufferedReader(fr);
+ } catch(FileNotFoundException excp) {
+ LogLog.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp);
+ }
+ }
+ }
+
+ LogLog.debug("<== openNextFile(" + mCurrentLogfile + ")");
+ }
+
+ private void closeCurrentFile() {
+ LogLog.debug("==> closeCurrentFile(" + mCurrentLogfile + ")");
+
+ if(mReader != null) {
+ try {
+ mReader.close();
+ } catch(IOException excp) {
+ // ignore
+ }
+ }
+ mReader = null;
+
+ archiveCurrentFile();
+
+ LogLog.debug("<== closeCurrentFile(" + mCurrentLogfile + ")");
+ }
+
+ private void archiveCurrentFile() {
+ if(mCurrentLogfile != null) {
+ File logFile = new File(mCurrentLogfile);
+ String archiveFilename = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory() + File.separator + logFile.getName());
+
+ try {
+ if(logFile.exists()) {
+ File archiveFile = new File(archiveFilename);
+
+ MiscUtil.createParents(archiveFile);
+
+ if(! logFile.renameTo(archiveFile)) {
+ // TODO: renameTo() does not work in all cases. in case of failure, copy the file contents to the destination and delete the file
+ }
+
+ // TODO: ensure no more than mFileLogBuffer.getArchiveFileCount() archive files are kept
+ }
+ } catch(Exception excp) {
+ LogLog.warn("archiveCurrentFile(): faile to move " + mCurrentLogfile + " to archive location " + archiveFilename, excp);
+ }
+ }
+ mCurrentLogfile = null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/src/main/java/org/apache/log4j/LogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/LogBuffer.java b/agents-common/src/main/java/org/apache/log4j/LogBuffer.java
new file mode 100644
index 0000000..799aacb
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/log4j/LogBuffer.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.log4j;
+
+
+public interface LogBuffer<T> {
+ public void start(LogDestination<T> destination);
+
+ public void stop();
+
+ boolean isAvailable();
+
+ public boolean add(T log);
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/be47e640/agents-common/src/main/java/org/apache/log4j/LogDestination.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/LogDestination.java b/agents-common/src/main/java/org/apache/log4j/LogDestination.java
new file mode 100644
index 0000000..fd4e8f6
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/log4j/LogDestination.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.log4j;
+
+
+public interface LogDestination<T> {
+ public void start();
+
+ public void stop();
+
+ boolean isAvailable();
+
+ public boolean add(T log);
+}
[07/10] git commit: ARGUS-5: added configurable retry interval
between attempts to open HDFS file.
Posted by ma...@apache.org.
ARGUS-5: added configurable retry interval between attempts to open HDFS
file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/c324e8df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/c324e8df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/c324e8df
Branch: refs/heads/master
Commit: c324e8dfa123091a49bff1b3d3f34675f232c7e2
Parents: 5ccf382
Author: mneethiraj <mn...@hortonworks.com>
Authored: Sun Sep 21 14:11:16 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Sun Sep 21 14:11:16 2014 -0700
----------------------------------------------------------------------
.../audit/provider/LocalFileLogBuffer.java | 83 ++++++++++----------
.../com/xasecure/audit/provider/MiscUtil.java | 2 +-
.../audit/provider/hdfs/HdfsAuditProvider.java | 8 +-
.../audit/provider/hdfs/HdfsLogDestination.java | 67 ++++++++++------
hbase-agent/conf/xasecure-audit.xml | 5 ++
hdfs-agent/conf/xasecure-audit.xml | 5 ++
hive-agent/conf/xasecure-audit.xml | 5 ++
knox-agent/conf/xasecure-audit.xml | 5 ++
storm-agent/conf/xasecure-audit.xml | 5 ++
9 files changed, 114 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 9acae11..280d02c 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -50,16 +50,16 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
private String mArchiveDirectory = null;
private int mArchiveFileCount = 10;
- private Writer mWriter = null;
- private String mCurrentFilename = null;
- private long mNextRolloverTime = 0;
+ private Writer mWriter = null;
+ private String mBufferFilename = null;
+ private long mNextRolloverTime = 0;
private Gson mGsonBuilder = null;
private DestinationDispatcherThread<T> mDispatcherThread = null;
public LocalFileLogBuffer() {
- mGsonBuilder = new GsonBuilder().setPrettyPrinting().create();
+ mGsonBuilder = new GsonBuilder().create();
}
public String getDirectory() {
@@ -161,11 +161,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
public synchronized boolean add(T log) {
boolean ret = false;
- long now = System.currentTimeMillis();
-
- if(now > mNextRolloverTime) {
- rollover();
- }
+ rolloverIfNeeded();
Writer writer = mWriter;
@@ -195,16 +191,16 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
closeFile();
- mCurrentFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+ mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
FileOutputStream ostream = null;
try {
- ostream = new FileOutputStream(mCurrentFilename, mIsAppend);
+ ostream = new FileOutputStream(mBufferFilename, mIsAppend);
} catch(Exception excp) {
- MiscUtil.createParents(new File(mCurrentFilename));
+ MiscUtil.createParents(new File(mBufferFilename));
try {
- ostream = new FileOutputStream(mCurrentFilename, mIsAppend);
+ ostream = new FileOutputStream(mBufferFilename, mIsAppend);
} catch(Exception ex) {
// ignore; error printed down
}
@@ -213,13 +209,13 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
mWriter = createWriter(ostream);
if(mWriter != null) {
- LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mCurrentFilename);
+ LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
} else {
- LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mCurrentFilename);
+ LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
- mCurrentFilename = null;
+ mBufferFilename = null;
}
LogLog.debug("<== LocalFileLogBuffer.openFile()");
@@ -237,11 +233,11 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
writer.flush();
writer.close();
} catch(IOException excp) {
- LogLog.warn("LocalFileLogBuffer: failed to close file " + mCurrentFilename, excp);
+ LogLog.warn("LocalFileLogBuffer: failed to close file " + mBufferFilename, excp);
}
if(mDispatcherThread != null) {
- mDispatcherThread.addLogfile(mCurrentFilename);
+ mDispatcherThread.addLogfile(mBufferFilename);
}
}
@@ -258,6 +254,14 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
LogLog.debug("<== LocalFileLogBuffer.rollover()");
}
+ private void rolloverIfNeeded() {
+ long now = System.currentTimeMillis();
+
+ if(now > mNextRolloverTime) {
+ rollover();
+ }
+ }
+
public OutputStreamWriter createWriter(OutputStream os ) {
OutputStreamWriter writer = null;
@@ -279,7 +283,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
}
boolean isCurrentFilename(String filename) {
- return mCurrentFilename != null && filename != null && filename.equals(mCurrentFilename);
+ return mBufferFilename != null && filename != null && filename.equals(mBufferFilename);
}
private String toJson(T log) {
@@ -395,6 +399,8 @@ class DestinationDispatcherThread<T> extends Thread {
}
private boolean sendCurrentFile() {
+ LogLog.debug("==> DestinationDispatcherThread.sendCurrentFile()");
+
boolean ret = false;
int destinationPollIntervalInMs = 1000;
@@ -418,6 +424,12 @@ class DestinationDispatcherThread<T> extends Thread {
closeCurrentFile();
+ if(!mStopThread) {
+ archiveCurrentFile();
+ }
+
+ LogLog.debug("<== DestinationDispatcherThread.sendCurrentFile()");
+
return ret;
}
@@ -452,12 +464,7 @@ class DestinationDispatcherThread<T> extends Thread {
} catch (IOException excp) {
LogLog.warn("getNextStringifiedLog.getNextLog(): failed to read from file " + mCurrentLogfile, excp);
}
-
- if(log == null) {
- closeCurrentFile();
- }
}
- LogLog.warn("READ: " + log);
return log;
}
@@ -465,21 +472,17 @@ class DestinationDispatcherThread<T> extends Thread {
private void openCurrentFile() {
LogLog.debug("==> openCurrentFile(" + mCurrentLogfile + ")");
- closeCurrentFile();
-
- while(mReader == null) {
- if(mCurrentLogfile != null) {
- try {
- FileInputStream inStr = new FileInputStream(mCurrentLogfile);
-
- InputStreamReader strReader = createReader(inStr);
-
- if(strReader != null) {
- mReader = new BufferedReader(strReader);
- }
- } catch(FileNotFoundException excp) {
- LogLog.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp);
+ if(mCurrentLogfile != null) {
+ try {
+ FileInputStream inStr = new FileInputStream(mCurrentLogfile);
+
+ InputStreamReader strReader = createReader(inStr);
+
+ if(strReader != null) {
+ mReader = new BufferedReader(strReader);
}
+ } catch(FileNotFoundException excp) {
+ LogLog.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp);
}
}
@@ -498,10 +501,6 @@ class DestinationDispatcherThread<T> extends Thread {
}
mReader = null;
- if(!mStopThread) {
- archiveCurrentFile();
- }
-
LogLog.debug("<== closeCurrentFile(" + mCurrentLogfile + ")");
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
index 6610210..c84fdf6 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -42,7 +42,7 @@ public class MiscUtil {
try {
hostName = InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException excp) {
- LogLog.warn("LocalFileLogBuffer", excp);
+ LogLog.warn("replaceHostname()", excp);
}
if(hostName == null) {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
index db8489c..e8b3922 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -15,9 +15,10 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
public void init(Map<String, String> properties) {
String encoding = properties.get("encoding");
- String hdfsDestinationDirectory = properties.get("destination.directroy");
- String hdfsDestinationFile = properties.get("destination.file");
- int hdfsDestinationRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.rollover.interval.seconds"), 24 * 60 * 60);
+ String hdfsDestinationDirectory = properties.get("destination.directroy");
+ String hdfsDestinationFile = properties.get("destination.file");
+ int hdfsDestinationRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.rollover.interval.seconds"), 24 * 60 * 60);
+ int hdfsDestinationOpenRetryIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.open.retry.interval.seconds"), 60);
String localFileBufferDirectory = properties.get("local.buffer.directroy");
String localFileBufferFile = properties.get("local.buffer.file");
@@ -31,6 +32,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
mHdfsDestination.setFile(hdfsDestinationFile);
mHdfsDestination.setEncoding(encoding);
mHdfsDestination.setRolloverIntervalSeconds(hdfsDestinationRolloverIntervalSeconds);
+ mHdfsDestination.setOpenRetryIntervalSeconds(hdfsDestinationOpenRetryIntervalSeconds);
LocalFileLogBuffer<AuditEventBase> mLocalFileBuffer = new LocalFileLogBuffer<AuditEventBase>();
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index eeb7574..7567962 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -20,7 +20,6 @@ package com.xasecure.audit.provider.hdfs;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
@@ -37,16 +36,18 @@ import com.xasecure.audit.provider.LogDestination;
import com.xasecure.audit.provider.MiscUtil;
public class HdfsLogDestination<T> implements LogDestination<T> {
- private String mDirectory = null;
- private String mFile = null;
- private String mEncoding = null;
- private boolean mIsAppend = true;
- private int mRolloverIntervalSeconds = 24 * 60 * 60;
-
- private OutputStreamWriter mWriter = null;
- private String mCurrentFilename = null;
- private long mNextRolloverTime = 0;
- private boolean mIsStopInProgress = false;
+ private String mDirectory = null;
+ private String mFile = null;
+ private String mEncoding = null;
+ private boolean mIsAppend = true;
+ private int mRolloverIntervalSeconds = 24 * 60 * 60;
+ private int mOpenRetryIntervalSeconds = 60;
+
+ private OutputStreamWriter mWriter = null;
+ private String mHdfsFilename = null;
+ private long mNextRolloverTime = 0;
+ private long mLastOpenFailedTime = 0;
+ private boolean mIsStopInProgress = false;
public HdfsLogDestination() {
}
@@ -83,6 +84,14 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
this.mRolloverIntervalSeconds = rolloverIntervalSeconds;
}
+ public int getOpenRetryIntervalSeconds() {
+ return mOpenRetryIntervalSeconds;
+ }
+
+ public void setOpenRetryIntervalSeconds(int minIntervalOpenRetrySeconds) {
+ this.mOpenRetryIntervalSeconds = minIntervalOpenRetrySeconds;
+ }
+
@Override
public void start() {
LogLog.debug("==> HdfsLogDestination.start()");
@@ -127,7 +136,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
public boolean sendStringified(String log) {
boolean ret = false;
- rolloverIfNeeded();
+ checkDestinationFileStatus();
OutputStreamWriter writer = mWriter;
@@ -138,6 +147,8 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
ret = true;
} catch (IOException excp) {
LogLog.warn("HdfsLogDestination.sendStringified(): write failed", excp);
+
+ closeFile();
}
}
@@ -149,7 +160,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
closeFile();
- mCurrentFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+ mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
FSDataOutputStream ostream = null;
FileSystem fileSystem = null;
@@ -157,14 +168,14 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
Configuration conf = null;
try {
- LogLog.debug("HdfsLogDestination.openFile(): opening file " + mCurrentFilename);
+ LogLog.debug("HdfsLogDestination.openFile(): opening file " + mHdfsFilename);
- URI uri = URI.create(mCurrentFilename);
+ URI uri = URI.create(mHdfsFilename);
// TODO: mechanism to XA-HDFS plugin to disable auditing of access checks to the current HDFS file
conf = new Configuration();
- pathLogfile = new Path(mCurrentFilename);
+ pathLogfile = new Path(mHdfsFilename);
fileSystem = FileSystem.get(uri, conf);
if(fileSystem.exists(pathLogfile)) {
@@ -174,7 +185,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
} catch(IOException excp) {
// append may not be supported by the filesystem. rename existing file and create a new one
String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_CREATE_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_CREATE_TIME_END);
- String movedFilename = appendToFilename(mCurrentFilename, fileSuffix);
+ String movedFilename = appendToFilename(mHdfsFilename, fileSuffix);
Path movedFilePath = new Path(movedFilename);
fileSystem.rename(pathLogfile, movedFilePath);
@@ -207,16 +218,18 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
mWriter = createWriter(ostream);
if(mWriter != null) {
- LogLog.debug("HdfsLogDestination.openFile(): opened file " + mCurrentFilename);
+ LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);
mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ mLastOpenFailedTime = 0;
} else {
- LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mCurrentFilename);
+ LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);
- mCurrentFilename = null;
+ mHdfsFilename = null;
+ mLastOpenFailedTime = System.currentTimeMillis();
}
- LogLog.debug("<== HdfsLogDestination.openFile(" + mCurrentFilename + ")");
+ LogLog.debug("<== HdfsLogDestination.openFile(" + mHdfsFilename + ")");
}
private void closeFile() {
@@ -232,7 +245,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
writer.close();
} catch(IOException excp) {
if(! mIsStopInProgress) { // during shutdown, the underlying FileSystem might already be closed; so don't print error details
- LogLog.warn("HdfsLogDestination: failed to close file " + mCurrentFilename, excp);
+ LogLog.warn("HdfsLogDestination: failed to close file " + mHdfsFilename, excp);
}
}
}
@@ -250,10 +263,14 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
LogLog.debug("<== HdfsLogDestination.rollover()");
}
- private void rolloverIfNeeded() {
+ private void checkDestinationFileStatus() {
long now = System.currentTimeMillis();
- if(now > mNextRolloverTime) {
+ if(mWriter == null) {
+ if(now > (mLastOpenFailedTime + (mOpenRetryIntervalSeconds * 1000))) {
+ openFile();
+ }
+ } else if(now > mNextRolloverTime) {
rollover();
}
}
@@ -266,7 +283,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
try {
writer = new OutputStreamWriter(os, mEncoding);
} catch(UnsupportedEncodingException excp) {
- LogLog.warn("LocalFileLogBuffer: failed to create output writer.", excp);
+ LogLog.warn("HdfsLogDestination.createWriter(): failed to create output writer.", excp);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/hbase-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit.xml b/hbase-agent/conf/xasecure-audit.xml
index f33b8ba..11597b3 100644
--- a/hbase-agent/conf/xasecure-audit.xml
+++ b/hbase-agent/conf/xasecure-audit.xml
@@ -136,6 +136,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
<value>/tmp/logs/hbase</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/hdfs-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit.xml b/hdfs-agent/conf/xasecure-audit.xml
index 1ae6f3b..6189cf2 100644
--- a/hdfs-agent/conf/xasecure-audit.xml
+++ b/hdfs-agent/conf/xasecure-audit.xml
@@ -117,6 +117,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
<value>/tmp/logs/hdfs</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/hive-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit.xml b/hive-agent/conf/xasecure-audit.xml
index eb951a4..73c3d74 100644
--- a/hive-agent/conf/xasecure-audit.xml
+++ b/hive-agent/conf/xasecure-audit.xml
@@ -136,6 +136,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
<value>/tmp/logs/hive</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/knox-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit.xml b/knox-agent/conf/xasecure-audit.xml
index 987a49d..d7d1a8d 100644
--- a/knox-agent/conf/xasecure-audit.xml
+++ b/knox-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
<value>/tmp/logs/knox</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/c324e8df/storm-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit.xml b/storm-agent/conf/xasecure-audit.xml
index ef0b27a..b103aca 100644
--- a/storm-agent/conf/xasecure-audit.xml
+++ b/storm-agent/conf/xasecure-audit.xml
@@ -136,6 +136,11 @@
</property>
<property>
+ <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
<name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
<value>/tmp/logs/storm</value>
</property>
[10/10] git commit: ARGUS-5: fix per review comments.
Posted by ma...@apache.org.
ARGUS-5: fix per review comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/cfec85a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/cfec85a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/cfec85a6
Branch: refs/heads/master
Commit: cfec85a65894eed3f1bcda2ee19281d66ad58bb4
Parents: 46584e4
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 11:28:34 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 11:28:34 2014 -0700
----------------------------------------------------------------------
.../src/main/java/com/xasecure/audit/provider/MiscUtil.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/cfec85a6/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
index 8aa7368..3a07710 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -28,6 +28,7 @@ public class MiscUtil {
try {
sGsonBuilder = new GsonBuilder().create();
} catch(Throwable excp) {
+ LogLog.warn("failed to create GsonBuilder object. stringigy() will return obj.toString(), instead of Json", excp);
}
}
[09/10] git commit: ARGUS-5: log filename generation logic updated to
use start-time of the current rollover period (instead of current time).
Posted by ma...@apache.org.
ARGUS-5: log filename generation logic updated to use start-time of the
current rollover period (instead of current time).
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/46584e44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/46584e44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/46584e44
Branch: refs/heads/master
Commit: 46584e44293f2bf379f5206c374ad7644a7b54b7
Parents: efd3501
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 11:13:34 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 11:13:34 2014 -0700
----------------------------------------------------------------------
.../audit/provider/LocalFileLogBuffer.java | 10 ++--
.../com/xasecure/audit/provider/MiscUtil.java | 49 +++++++++++---------
.../audit/provider/hdfs/HdfsLogDestination.java | 6 ++-
hbase-agent/conf/xasecure-audit.xml | 4 +-
hbase-agent/scripts/install.properties | 4 +-
hdfs-agent/conf/xasecure-audit.xml | 4 +-
hdfs-agent/scripts/install.properties | 4 +-
hive-agent/conf/xasecure-audit.xml | 4 +-
hive-agent/scripts/install.properties | 4 +-
knox-agent/conf/xasecure-audit.xml | 4 +-
knox-agent/scripts/install.properties | 4 +-
storm-agent/conf/xasecure-audit.xml | 4 +-
storm-agent/scripts/install.properties | 4 +-
13 files changed, 58 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 3e715a9..753c2e4 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -183,9 +183,11 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
private synchronized void openFile() {
LogLog.debug("==> LocalFileLogBuffer.openFile()");
+ long currentRolloverStartTime = MiscUtil.getCurrentRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+
closeFile();
- mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+ mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, currentRolloverStartTime);
FileOutputStream ostream = null;
try {
@@ -365,7 +367,7 @@ class DestinationDispatcherThread<T> extends Thread {
private void init() {
LogLog.debug("==> DestinationDispatcherThread.init()");
- String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory());
+ String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory(), 0);
File directory = new File(dirName);
if(directory.exists() && directory.isDirectory()) {
@@ -495,8 +497,8 @@ class DestinationDispatcherThread<T> extends Thread {
private void archiveCurrentFile() {
if(mCurrentLogfile != null) {
File logFile = new File(mCurrentLogfile);
- String archiveDirName = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory());
- String archiveFilename = archiveDirName + File.separator + MiscUtil.replaceTokens(logFile.getName());
+ String archiveDirName = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory(), 0);
+ String archiveFilename = archiveDirName + File.separator +logFile.getName();
try {
if(logFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
index 76b367a..8aa7368 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -5,7 +5,6 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.rmi.dgc.VMID;
import java.text.SimpleDateFormat;
-import java.util.Date;
import org.apache.log4j.helpers.LogLog;
@@ -13,11 +12,11 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
public class MiscUtil {
- public static final String TOKEN_HOSTNAME = "%hostname%";
- public static final String TOKEN_APP_INSTANCE = "%app-instance%";
- public static final String TOKEN_CREATE_TIME_START = "%create-time:";
- public static final String TOKEN_CREATE_TIME_END = "%";
- public static final String ESCAPE_STR = "\\";
+ public static final String TOKEN_HOSTNAME = "%hostname%";
+ public static final String TOKEN_JVM_INSTANCE = "%jvm-instance%";
+ public static final String TOKEN_TIME_START = "%time:";
+ public static final String TOKEN_TIME_END = "%";
+ public static final String ESCAPE_STR = "\\";
static VMID sJvmID = new VMID();
@@ -32,14 +31,14 @@ public class MiscUtil {
}
}
- public static String replaceTokens(String str) {
+ public static String replaceTokens(String str, long time) {
if(str == null) {
return str;
}
str = replaceHostname(str);
- str = replaceAppInstance(str);
- str = replaceCreateTime(str);
+ str = replaceJvmInstance(str);
+ str = replaceTime(str, time);
return str;
}
@@ -64,36 +63,38 @@ public class MiscUtil {
return str.replace(TOKEN_HOSTNAME, hostName);
}
- public static String replaceAppInstance(String str) {
- if(!str.contains(TOKEN_APP_INSTANCE)) {
+ public static String replaceJvmInstance(String str) {
+ if(!str.contains(TOKEN_JVM_INSTANCE)) {
return str;
}
- String appInstance = Integer.toString(Math.abs(sJvmID.hashCode()));
+ String jvmInstance = Integer.toString(Math.abs(sJvmID.hashCode()));
- return str.replace(TOKEN_APP_INSTANCE, appInstance);
+ return str.replace(TOKEN_JVM_INSTANCE, jvmInstance);
}
- public static String replaceCreateTime(String str) {
- Date now = new Date();
+ public static String replaceTime(String str, long time) {
+ if(time <= 0) {
+ time = System.currentTimeMillis();
+ }
- while(str.contains(TOKEN_CREATE_TIME_START)) {
- int tagStartPos = str.indexOf(TOKEN_CREATE_TIME_START);
- int tagEndPos = str.indexOf(TOKEN_CREATE_TIME_END, tagStartPos + TOKEN_CREATE_TIME_START.length());
+ while(str.contains(TOKEN_TIME_START)) {
+ int tagStartPos = str.indexOf(TOKEN_TIME_START);
+ int tagEndPos = str.indexOf(TOKEN_TIME_END, tagStartPos + TOKEN_TIME_START.length());
if(tagEndPos <= tagStartPos) {
break;
}
String tag = str.substring(tagStartPos, tagEndPos+1);
- String dtFormat = tag.substring(TOKEN_CREATE_TIME_START.length(), tag.lastIndexOf(TOKEN_CREATE_TIME_END));
+ String dtFormat = tag.substring(TOKEN_TIME_START.length(), tag.lastIndexOf(TOKEN_TIME_END));
String replaceStr = "";
if(dtFormat != null) {
SimpleDateFormat sdf = new SimpleDateFormat(dtFormat);
- replaceStr = sdf.format(now);
+ replaceStr = sdf.format(time);
}
str = str.replace(tag, replaceStr);
@@ -134,6 +135,10 @@ public class MiscUtil {
}
}
+ public static long getCurrentRolloverStartTime(long nextRolloverTime, long interval) {
+ return (nextRolloverTime <= interval) ? System.currentTimeMillis() : nextRolloverTime - interval;
+ }
+
public static int parseInteger(String str, int defValue) {
int ret = defValue;
@@ -152,7 +157,9 @@ public class MiscUtil {
String ret = null;
if(log != null) {
- if(MiscUtil.sGsonBuilder != null) {
+ if(log instanceof String) {
+ ret = (String)log;
+ } else if(MiscUtil.sGsonBuilder != null) {
ret = MiscUtil.sGsonBuilder.toJson(log);
} else {
ret = log.toString();
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index fbc0431..defb39f 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -158,9 +158,11 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
private void openFile() {
LogLog.debug("==> HdfsLogDestination.openFile()");
+ long currentRolloverStartTime = MiscUtil.getCurrentRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+
closeFile();
- mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+ mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, currentRolloverStartTime);
FSDataOutputStream ostream = null;
FileSystem fileSystem = null;
@@ -184,7 +186,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
ostream = fileSystem.append(pathLogfile);
} catch(IOException excp) {
// append may not be supported by the filesystem. rename existing file and create a new one
- String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_CREATE_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_CREATE_TIME_END);
+ String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_TIME_END, currentRolloverStartTime);
String movedFilename = appendToFilename(mHdfsFilename, fileSuffix);
Path movedFilePath = new Path(movedFilename);
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/hbase-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit.xml b/hbase-agent/conf/xasecure-audit.xml
index 11597b3..93f06d1 100644
--- a/hbase-agent/conf/xasecure-audit.xml
+++ b/hbase-agent/conf/xasecure-audit.xml
@@ -122,7 +122,7 @@
<property>
<name>xasecure.audit.hdfs.config.destination.directroy</name>
- <value>hdfs://namenodehost:8020/audit/hbase/%create-time:yyyyMMdd%</value>
+ <value>hdfs://namenodehost:8020/audit/hbase/%time:yyyyMMdd%</value>
</property>
<property>
@@ -147,7 +147,7 @@
<property>
<name>xasecure.audit.hdfs.config.local.buffer.file</name>
- <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ <value>%time:yyyyMMdd-HHmm.ss%.log</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index bfc5292..337b3f5 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -112,12 +112,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
# Audit to HDFS
#
XAAUDIT.HDFS.IS_ENABLED=false
-XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hbase/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hbase/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hbase
-XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hbase
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/hdfs-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit.xml b/hdfs-agent/conf/xasecure-audit.xml
index 6189cf2..e2b8f9d 100644
--- a/hdfs-agent/conf/xasecure-audit.xml
+++ b/hdfs-agent/conf/xasecure-audit.xml
@@ -103,7 +103,7 @@
<property>
<name>xasecure.audit.hdfs.config.destination.directroy</name>
- <value>hdfs://namenodehost:8020/audit/hdfs/%create-time:yyyyMMdd%</value>
+ <value>hdfs://namenodehost:8020/audit/hdfs/%time:yyyyMMdd%</value>
</property>
<property>
@@ -128,7 +128,7 @@
<property>
<name>xasecure.audit.hdfs.config.local.buffer.file</name>
- <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ <value>%time:yyyyMMdd-HHmm.ss%.log</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 73643b8..5cfa476 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -105,12 +105,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
# Audit to HDFS
#
XAAUDIT.HDFS.IS_ENABLED=false
-XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hdfs/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hdfs/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hdfs
-XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hdfs
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/hive-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit.xml b/hive-agent/conf/xasecure-audit.xml
index 73c3d74..319ae09 100644
--- a/hive-agent/conf/xasecure-audit.xml
+++ b/hive-agent/conf/xasecure-audit.xml
@@ -122,7 +122,7 @@
<property>
<name>xasecure.audit.hdfs.config.destination.directroy</name>
- <value>hdfs://namenodehost:8020/audit/hive/%create-time:yyyyMMdd%</value>
+ <value>hdfs://namenodehost:8020/audit/hive/%time:yyyyMMdd%</value>
</property>
<property>
@@ -147,7 +147,7 @@
<property>
<name>xasecure.audit.hdfs.config.local.buffer.file</name>
- <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ <value>%time:yyyyMMdd-HHmm.ss%.log</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 436c53d..4cbdde0 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -112,12 +112,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
# Audit to HDFS
#
XAAUDIT.HDFS.IS_ENABLED=false
-XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hive/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hive/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hive
-XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hive
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/knox-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit.xml b/knox-agent/conf/xasecure-audit.xml
index d7d1a8d..4a7303e 100644
--- a/knox-agent/conf/xasecure-audit.xml
+++ b/knox-agent/conf/xasecure-audit.xml
@@ -117,7 +117,7 @@
<property>
<name>xasecure.audit.hdfs.config.destination.directroy</name>
- <value>hdfs://namenodehost:8020/audit/knox/%create-time:yyyyMMdd%</value>
+ <value>hdfs://namenodehost:8020/audit/knox/%time:yyyyMMdd%</value>
</property>
<property>
@@ -142,7 +142,7 @@
<property>
<name>xasecure.audit.hdfs.config.local.buffer.file</name>
- <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ <value>%time:yyyyMMdd-HHmm.ss%.log</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index 3f9aa33..1f0e01e 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -98,12 +98,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
# Audit to HDFS
#
XAAUDIT.HDFS.IS_ENABLED=false
-XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/knox/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/knox/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/knox
-XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/knox
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/storm-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit.xml b/storm-agent/conf/xasecure-audit.xml
index b103aca..f8c07d2 100644
--- a/storm-agent/conf/xasecure-audit.xml
+++ b/storm-agent/conf/xasecure-audit.xml
@@ -122,7 +122,7 @@
<property>
<name>xasecure.audit.hdfs.config.destination.directroy</name>
- <value>hdfs://namenodehost:8020/audit/storm/%create-time:yyyyMMdd%</value>
+ <value>hdfs://namenodehost:8020/audit/storm/%time:yyyyMMdd%</value>
</property>
<property>
@@ -147,7 +147,7 @@
<property>
<name>xasecure.audit.hdfs.config.local.buffer.file</name>
- <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ <value>%time:yyyyMMdd-HHmm.ss%.log</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/46584e44/storm-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index 8e0ec8f..c1e3e44 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -102,12 +102,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
# Audit to HDFS
#
XAAUDIT.HDFS.IS_ENABLED=false
-XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/storm/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/storm/%time:yyyyMMdd%
XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/storm
-XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/storm
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
[08/10] git commit: ARGUS-5: installation updates to configure HDFS
audit provider. Added an option to enable/disable audit to database.
Posted by ma...@apache.org.
ARGUS-5: installation updates to configure HDFS audit provider. Added an
option to enable/disable audit to database.
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/efd3501d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/efd3501d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/efd3501d
Branch: refs/heads/master
Commit: efd3501dcc79c5a4985b881fbb36b84e374bae63
Parents: c324e8d
Author: mneethiraj <mn...@hortonworks.com>
Authored: Mon Sep 22 00:09:21 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Mon Sep 22 00:09:21 2014 -0700
----------------------------------------------------------------------
.../audit/provider/LocalFileLogBuffer.java | 16 ++-------
.../com/xasecure/audit/provider/MiscUtil.java | 37 ++++++++++++++++----
.../audit/provider/hdfs/HdfsLogDestination.java | 2 +-
hbase-agent/conf/xasecure-audit-changes.cfg | 12 +++++++
hbase-agent/scripts/install.properties | 16 +++++++++
hdfs-agent/conf/xasecure-audit-changes.cfg | 12 +++++++
hdfs-agent/scripts/install.properties | 16 +++++++++
hive-agent/conf/xasecure-audit-changes.cfg | 12 +++++++
hive-agent/scripts/install.properties | 16 +++++++++
knox-agent/conf/xasecure-audit-changes.cfg | 14 +++++++-
knox-agent/scripts/install.properties | 16 +++++++++
storm-agent/conf/xasecure-audit-changes.cfg | 12 +++++++
storm-agent/scripts/install.properties | 16 +++++++++
13 files changed, 175 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 280d02c..3e715a9 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -37,9 +37,6 @@ import java.util.TreeSet;
import org.apache.log4j.helpers.LogLog;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
public class LocalFileLogBuffer<T> implements LogBuffer<T> {
private String mDirectory = null;
@@ -54,12 +51,9 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
private String mBufferFilename = null;
private long mNextRolloverTime = 0;
- private Gson mGsonBuilder = null;
-
private DestinationDispatcherThread<T> mDispatcherThread = null;
public LocalFileLogBuffer() {
- mGsonBuilder = new GsonBuilder().create();
}
public String getDirectory() {
@@ -167,8 +161,8 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
if(writer != null) {
try {
- String msg = toJson(log);
-
+ String msg = MiscUtil.stringify(log);
+
if(msg.contains(MiscUtil.LINE_SEPARATOR)) {
msg = msg.replace(MiscUtil.LINE_SEPARATOR, MiscUtil.ESCAPE_STR + MiscUtil.LINE_SEPARATOR);
}
@@ -285,12 +279,6 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T> {
boolean isCurrentFilename(String filename) {
return mBufferFilename != null && filename != null && filename.equals(mBufferFilename);
}
-
- private String toJson(T log) {
- String jsonString = mGsonBuilder.toJson(log) ;
-
- return jsonString;
- }
@Override
public String toString() {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
index c84fdf6..76b367a 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -9,17 +9,29 @@ import java.util.Date;
import org.apache.log4j.helpers.LogLog;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
public class MiscUtil {
public static final String TOKEN_HOSTNAME = "%hostname%";
public static final String TOKEN_APP_INSTANCE = "%app-instance%";
public static final String TOKEN_CREATE_TIME_START = "%create-time:";
public static final String TOKEN_CREATE_TIME_END = "%";
public static final String ESCAPE_STR = "\\";
-
+
static VMID sJvmID = new VMID();
-
+
public static String LINE_SEPARATOR = System.getProperty("line.separator");
+ private static Gson sGsonBuilder = null;
+
+ static {
+ try {
+ sGsonBuilder = new GsonBuilder().create();
+ } catch(Throwable excp) {
+ }
+ }
+
public static String replaceTokens(String str) {
if(str == null) {
return str;
@@ -121,10 +133,10 @@ public class MiscUtil {
return lastRolloverTime;
}
}
-
+
public static int parseInteger(String str, int defValue) {
int ret = defValue;
-
+
if(str != null) {
try {
ret = Integer.parseInt(str);
@@ -132,8 +144,21 @@ public class MiscUtil {
// ignore
}
}
-
+
return ret;
}
-}
+ public static <T> String stringify(T log) {
+ String ret = null;
+
+ if(log != null) {
+ if(MiscUtil.sGsonBuilder != null) {
+ ret = MiscUtil.sGsonBuilder.toJson(log);
+ } else {
+ ret = log.toString();
+ }
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index 7567962..fbc0431 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -142,7 +142,7 @@ public class HdfsLogDestination<T> implements LogDestination<T> {
if(writer != null) {
try {
- writer.write(log);
+ writer.write(log + MiscUtil.LINE_SEPARATOR);
ret = true;
} catch (IOException excp) {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/hbase-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit-changes.cfg b/hbase-agent/conf/xasecure-audit-changes.cfg
index dd4b70b..66beb58 100644
--- a/hbase-agent/conf/xasecure-audit-changes.cfg
+++ b/hbase-agent/conf/xasecure-audit-changes.cfg
@@ -1,6 +1,18 @@
+xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT.DB.JDBC_URL% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists
xasecure.audit.repository.name %REPOSITORY_NAME% mod create-if-not-exists
xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% mod create-if-not-exists
+
+xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index fa963ec..bfc5292 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -103,5 +103,21 @@ UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
# XAAUDIT.DB.JDBC_URL=jdbc:oracle:thin:@//127.0.0.1:1521/XE
# XAAUDIT.DB.JDBC_DRIVER=oracle.jdbc.OracleDriver
+XAAUDIT.DB.IS_ENABLED=true
XAAUDIT.DB.JDBC_URL=jdbc:mysql://localhost:3306/xasecure
XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
+
+
+#
+# Audit to HDFS
+#
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hbase/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hbase
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hbase
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/hdfs-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit-changes.cfg b/hdfs-agent/conf/xasecure-audit-changes.cfg
index dd4b70b..66beb58 100644
--- a/hdfs-agent/conf/xasecure-audit-changes.cfg
+++ b/hdfs-agent/conf/xasecure-audit-changes.cfg
@@ -1,6 +1,18 @@
+xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT.DB.JDBC_URL% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists
xasecure.audit.repository.name %REPOSITORY_NAME% mod create-if-not-exists
xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% mod create-if-not-exists
+
+xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 07f181a..73643b8 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -96,5 +96,21 @@ SSL_TRUSTSTORE_PASSWORD=changeit
# XAAUDIT.DB.JDBC_URL=jdbc:oracle:thin:@//127.0.0.1:1521/XE
# XAAUDIT.DB.JDBC_DRIVER=oracle.jdbc.OracleDriver
+XAAUDIT.DB.IS_ENABLED=true
XAAUDIT.DB.JDBC_URL=jdbc:mysql://localhost:3306/xasecure
XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
+
+
+#
+# Audit to HDFS
+#
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hdfs/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hdfs
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hdfs
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/hive-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit-changes.cfg b/hive-agent/conf/xasecure-audit-changes.cfg
index 42ed849..0ae94fb 100644
--- a/hive-agent/conf/xasecure-audit-changes.cfg
+++ b/hive-agent/conf/xasecure-audit-changes.cfg
@@ -1,6 +1,18 @@
+xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT.DB.JDBC_URL% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists
xasecure.audit.repository.name %REPOSITORY_NAME% mod create-if-not-exists
xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% mod create-if-not-exists
+
+xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 1ef9c22..436c53d 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -103,5 +103,21 @@ UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
# XAAUDIT.DB.JDBC_URL=jdbc:oracle:thin:@//127.0.0.1:1521/XE
# XAAUDIT.DB.JDBC_DRIVER=oracle.jdbc.OracleDriver
+XAAUDIT.DB.IS_ENABLED=true
XAAUDIT.DB.JDBC_URL=jdbc:mysql://localhost:3306/xasecure
XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
+
+
+#
+# Audit to HDFS
+#
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hive/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hive
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/knox-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit-changes.cfg b/knox-agent/conf/xasecure-audit-changes.cfg
index 22a4da7..9e0340e 100644
--- a/knox-agent/conf/xasecure-audit-changes.cfg
+++ b/knox-agent/conf/xasecure-audit-changes.cfg
@@ -1,5 +1,17 @@
+xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT.DB.JDBC_URL% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists
xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
xasecure.audit.repository.name %REPOSITORY_NAME% mod create-if-not-exists
-xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% mod create-if-not-exists
\ No newline at end of file
+xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% mod create-if-not-exists
+
+xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index f84168d..3f9aa33 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -89,5 +89,21 @@ SSL_TRUSTSTORE_PASSWORD=changeit
# XAAUDIT.DB.JDBC_URL=jdbc:oracle:thin:@//127.0.0.1:1521/XE
# XAAUDIT.DB.JDBC_DRIVER=oracle.jdbc.OracleDriver
+XAAUDIT.DB.IS_ENABLED=true
XAAUDIT.DB.JDBC_URL=jdbc:mysql://localhost:3306/xasecure
XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
+
+
+#
+# Audit to HDFS
+#
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/knox/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/knox
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/knox
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/storm-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit-changes.cfg b/storm-agent/conf/xasecure-audit-changes.cfg
index f0c1050..68a2484 100644
--- a/storm-agent/conf/xasecure-audit-changes.cfg
+++ b/storm-agent/conf/xasecure-audit-changes.cfg
@@ -1,6 +1,18 @@
+xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT.DB.JDBC_URL% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists
xasecure.audit.repository.name %REPOSITORY_NAME% mod create-if-not-exists
xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% mod create-if-not-exists
+
+xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/efd3501d/storm-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index 359f66b..8e0ec8f 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -93,5 +93,21 @@ SSL_TRUSTSTORE_PASSWORD=changeit
# XAAUDIT.DB.JDBC_URL=jdbc:oracle:thin:@//127.0.0.1:1521/XE
# XAAUDIT.DB.JDBC_DRIVER=oracle.jdbc.OracleDriver
+XAAUDIT.DB.IS_ENABLED=true
XAAUDIT.DB.JDBC_URL=jdbc:mysql://localhost:3306/xasecure
XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
+
+
+#
+# Audit to HDFS
+#
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/storm/%create-time:yyyyMMdd%
+XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/storm
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%create-time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/storm
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
[03/10] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-argus
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-argus
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/2fde3325
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/2fde3325
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/2fde3325
Branch: refs/heads/master
Commit: 2fde332537b74a74e0bad8ac1a0debbe40570c56
Parents: f7c934c 4eacc7a
Author: mneethiraj <mn...@hortonworks.com>
Authored: Fri Sep 19 17:49:51 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Fri Sep 19 17:49:51 2014 -0700
----------------------------------------------------------------------
.../audit/provider/Log4jAuditProvider.java | 7 +-
.../hadoop/log/HdfsFileAppender.java | 1360 ++++++++++++++++++
.../com/xasecure/pdp/config/ConfigWatcher.java | 27 +-
.../xasecure/pdp/config/PolicyRefresher.java | 70 +-
.../xasecure/server/tomcat/EmbededServer.java | 46 +-
hbase-agent/scripts/install.sh | 2 +
hive-agent/scripts/install.sh | 2 +
7 files changed, 1477 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/2fde3325/agents-audit/src/main/java/com/xasecure/audit/provider/Log4jAuditProvider.java
----------------------------------------------------------------------
[06/10] git commit: ARGUS-5: added support for sending audit logs to
HDFS
Posted by ma...@apache.org.
ARGUS-5: added support for sending audit logs to HDFS
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/5ccf382d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/5ccf382d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/5ccf382d
Branch: refs/heads/master
Commit: 5ccf382d13d046179d2b1bbd20449917cfc7b8c1
Parents: 886e9c2
Author: mneethiraj <mn...@hortonworks.com>
Authored: Sun Sep 21 00:37:28 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Sun Sep 21 00:37:28 2014 -0700
----------------------------------------------------------------------
.../audit/provider/AuditProviderFactory.java | 110 ++--
.../audit/provider/BufferedAuditProvider.java | 65 ++
.../audit/provider/DummyAuditProvider.java | 13 +-
.../audit/provider/LocalFileLogBuffer.java | 601 +++++++++++++++++++
.../com/xasecure/audit/provider/LogBuffer.java | 30 +
.../xasecure/audit/provider/LogDestination.java | 32 +
.../com/xasecure/audit/provider/MiscUtil.java | 139 +++++
.../audit/provider/hdfs/HdfsAuditProvider.java | 48 ++
.../audit/provider/hdfs/HdfsLogDestination.java | 315 ++++++++++
hbase-agent/conf/xasecure-audit.xml | 72 +++
hdfs-agent/conf/xasecure-audit.xml | 141 ++++-
hive-agent/conf/xasecure-audit.xml | 72 +++
knox-agent/conf/xasecure-audit.xml | 72 +++
storm-agent/conf/xasecure-audit.xml | 72 +++
14 files changed, 1708 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
index 7bed0f7..bb31e7c 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
@@ -27,6 +27,7 @@ import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import com.xasecure.audit.provider.hdfs.HdfsAuditProvider;
import com.xasecure.authorization.hadoop.utils.XaSecureCredentialProvider;
@@ -41,22 +42,32 @@ public class AuditProviderFactory {
private static final Log LOG = LogFactory.getLog(AuditProviderFactory.class);
- private static final String AUDIT_JPA_CONFIG_PROP_PREFIX = "xasecure.audit.jpa.";
private static final String AUDIT_IS_ENABLED_PROP = "xasecure.audit.is.enabled" ;
- private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ;
- private static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async" ;
- private static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size" ;
- private static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms";
+
private static final String AUDIT_DB_IS_ENABLED_PROP = "xasecure.audit.db.is.enabled" ;
- private static final String AUDIT_DB_IS_ASYNC_PROP = "xasecure.audit.db.is.async" ;
+ private static final String AUDIT_DB_IS_ASYNC_PROP = "xasecure.audit.db.is.async";
private static final String AUDIT_DB_MAX_QUEUE_SIZE_PROP = "xasecure.audit.db.async.max.queue.size" ;
- private static final String AUDIT_DB_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.db.async.max.flush.interval.ms";
private static final String AUDIT_DB_RESUME_QUEUE_SIZE__PROP = "xasecure.audit.db.async.resume.queue.size" ;
+ private static final String AUDIT_DB_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.db.async.max.flush.interval.ms";
private static final String AUDIT_DB_BATCH_SIZE_PROP = "xasecure.audit.db.batch.size" ;
+ private static final String AUDIT_JPA_CONFIG_PROP_PREFIX = "xasecure.audit.jpa.";
private static final String AUDIT_DB_CREDENTIAL_PROVIDER_FILE = "xasecure.audit.credential.provider.file";
private static final String AUDIT_DB_CREDENTIAL_PROVIDER_ALIAS = "auditDBCred";
private static final String AUDIT_JPA_JDBC_PASSWORD = "javax.persistence.jdbc.password";
+ private static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
+ private static final String AUDIT_HDFS_IS_ASYNC_PROP = "xasecure.audit.hdfs.is.async";
+ private static final String AUDIT_HDFS_MAX_QUEUE_SIZE_PROP = "xasecure.audit.hdfs.async.max.queue.size" ;
+ private static final String AUDIT_HDFS_RESUME_QUEUE_SIZE__PROP = "xasecure.audit.hdfs.async.resume.queue.size" ;
+ private static final String AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.hdfs.async.max.flush.interval.ms";
+ private static final String AUDIT_HDFS_CONFIG_PREFIX_PROP = "xasecure.audit.hdfs.config.";
+
+ private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ;
+ private static final String AUDIT_LOG4J_IS_ASYNC_PROP = "xasecure.audit.log4j.is.async";
+ private static final String AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP = "xasecure.audit.log4j.async.max.queue.size" ;
+ private static final String AUDIT_LOG4J_RESUME_QUEUE_SIZE__PROP = "xasecure.audit.log4j.async.resume.queue.size" ;
+ private static final String AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.log4j.async.max.flush.interval.ms";
+
private static AuditProviderFactory sFactory;
private AuditProvider mProvider = null;
@@ -89,32 +100,35 @@ public class AuditProviderFactory {
public void init(Properties props) {
LOG.info("AuditProviderFactory: initializing..");
-
+
boolean isEnabled = getBooleanProperty(props, AUDIT_IS_ENABLED_PROP, false);
- boolean isAuditToLog4jEnabled = getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
- boolean isAuditToLog4jAsync = getBooleanProperty(props, AUDIT_LOG4J_IS_ASYNC_PROP, false);
boolean isAuditToDbEnabled = getBooleanProperty(props, AUDIT_DB_IS_ENABLED_PROP, false);
- boolean isAuditToDbAsync = getBooleanProperty(props, AUDIT_DB_IS_ASYNC_PROP, false);
-
- List<AuditProvider> providers = new ArrayList<AuditProvider>();
-
+ boolean isAuditToHdfsEnabled = getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
+ boolean isAuditToLog4jEnabled = getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
- if(!isEnabled || (!isAuditToDbEnabled && !isAuditToLog4jEnabled)) {
+ if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToLog4jEnabled)) {
LOG.info("AuditProviderFactory: Audit not enabled..");
-
+
mProvider = getDefaultProvider();
return;
}
-
+
+ List<AuditProvider> providers = new ArrayList<AuditProvider>();
+
if(isAuditToDbEnabled) {
-
-
- Map<String, String> jpaInitProperties = getJpaProperties(props);
-
+ Map<String, String> jpaInitProperties = getPropertiesWithPrefix(props, AUDIT_JPA_CONFIG_PROP_PREFIX);
+
+ String jdbcPassword = getCredentialString(getStringProperty(props, AUDIT_DB_CREDENTIAL_PROVIDER_FILE), AUDIT_DB_CREDENTIAL_PROVIDER_ALIAS);
+
+ if(jdbcPassword != null && !jdbcPassword.isEmpty()) {
+ jpaInitProperties.put(AUDIT_JPA_JDBC_PASSWORD, jdbcPassword);
+ }
+
LOG.info("AuditProviderFactory: found " + jpaInitProperties.size() + " Audit JPA properties");
- int dbBatchSize = getIntProperty(props, AUDIT_DB_BATCH_SIZE_PROP, 1000);
+ int dbBatchSize = getIntProperty(props, AUDIT_DB_BATCH_SIZE_PROP, 1000);
+ boolean isAuditToDbAsync = getBooleanProperty(props, AUDIT_DB_IS_ASYNC_PROP, false);
if(! isAuditToDbAsync) {
dbBatchSize = 1; // Batching not supported in sync mode; need to address multiple threads making audit calls
@@ -142,17 +156,51 @@ public class AuditProviderFactory {
}
}
+ if(isAuditToHdfsEnabled) {
+ Map<String, String> hdfsInitProperties = getPropertiesWithPrefix(props, AUDIT_HDFS_CONFIG_PREFIX_PROP);
+
+ LOG.info("AuditProviderFactory: found " + hdfsInitProperties.size() + " Audit HDFS properties");
+
+ HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
+
+ hdfsProvider.init(hdfsInitProperties);
+
+ boolean isAuditToHdfsAsync = getBooleanProperty(props, AUDIT_HDFS_IS_ASYNC_PROP, false);
+
+ if(isAuditToHdfsAsync) {
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider();
+
+ int maxQueueSize = getIntProperty(props, AUDIT_HDFS_MAX_QUEUE_SIZE_PROP, -1);
+ int maxFlushInterval = getIntProperty(props, AUDIT_HDFS_MAX_FLUSH_INTERVAL_PROP, -1);
+ int resumeQueueSize = getIntProperty(props, AUDIT_HDFS_RESUME_QUEUE_SIZE__PROP, 0);
+
+ asyncProvider.setMaxQueueSize(maxQueueSize);
+ asyncProvider.setMaxFlushInterval(maxFlushInterval);
+ asyncProvider.setResumeQueueSize(resumeQueueSize);
+
+ asyncProvider.addAuditProvider(hdfsProvider);
+
+ providers.add(asyncProvider);
+ } else {
+ providers.add(hdfsProvider);
+ }
+ }
+
if(isAuditToLog4jEnabled) {
Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
+
+ boolean isAuditToLog4jAsync = getBooleanProperty(props, AUDIT_LOG4J_IS_ASYNC_PROP, false);
if(isAuditToLog4jAsync) {
AsyncAuditProvider asyncProvider = new AsyncAuditProvider();
int maxQueueSize = getIntProperty(props, AUDIT_LOG4J_MAX_QUEUE_SIZE_PROP, -1);
int maxFlushInterval = getIntProperty(props, AUDIT_LOG4J_MAX_FLUSH_INTERVAL_PROP, -1);
+ int resumeQueueSize = getIntProperty(props, AUDIT_LOG4J_RESUME_QUEUE_SIZE__PROP, 0);
asyncProvider.setMaxQueueSize(maxQueueSize);
asyncProvider.setMaxFlushInterval(maxFlushInterval);
+ asyncProvider.setResumeQueueSize(resumeQueueSize);
asyncProvider.addAuditProvider(log4jProvider);
@@ -181,8 +229,8 @@ public class AuditProviderFactory {
Runtime.getRuntime().addShutdownHook(jvmShutdownHook);
}
- private Map<String, String> getJpaProperties(Properties props) {
- Map<String, String> jpaInitProperties = new HashMap<String, String>();
+ private Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) {
+ Map<String, String> prefixedProperties = new HashMap<String, String>();
for(String key : props.stringPropertyNames()) {
if(key == null) {
@@ -191,25 +239,19 @@ public class AuditProviderFactory {
String val = props.getProperty(key);
- if(key.startsWith(AuditProviderFactory.AUDIT_JPA_CONFIG_PROP_PREFIX)) {
- key = key.substring(AuditProviderFactory.AUDIT_JPA_CONFIG_PROP_PREFIX.length());
+ if(key.startsWith(prefix)) {
+ key = key.substring(prefix.length());
if(key == null) {
continue;
}
- jpaInitProperties.put(key, val);
+ prefixedProperties.put(key, val);
}
}
- String jdbcPassword = getCredentialString(getStringProperty(props,AUDIT_DB_CREDENTIAL_PROVIDER_FILE), AUDIT_DB_CREDENTIAL_PROVIDER_ALIAS);
-
- if(jdbcPassword != null && !jdbcPassword.isEmpty()) {
- jpaInitProperties.put(AUDIT_JPA_JDBC_PASSWORD, jdbcPassword);
- }
-
- return jpaInitProperties;
+ return prefixedProperties;
}
private boolean getBooleanProperty(Properties props, String propName, boolean defValue) {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
new file mode 100644
index 0000000..9b8cb40
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xasecure.audit.provider;
+
+import com.xasecure.audit.model.AuditEventBase;
+
+public abstract class BufferedAuditProvider implements AuditProvider {
+ private LogBuffer<AuditEventBase> mBuffer = null;
+ private LogDestination<AuditEventBase> mDestination = null;
+
+
+ @Override
+ public void log(AuditEventBase event) {
+ mBuffer.add(event);
+ }
+
+ @Override
+ public void start() {
+ mBuffer.start(mDestination);
+ }
+
+ @Override
+ public void stop() {
+ mBuffer.stop();
+ }
+
+ @Override
+ public void waitToComplete() {
+ }
+
+ @Override
+ public boolean isFlushPending() {
+ return false;
+ }
+
+ @Override
+ public long getLastFlushTime() {
+ return 0;
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ protected void setBufferAndDestination(LogBuffer<AuditEventBase> buffer,
+ LogDestination<AuditEventBase> destination) {
+ mBuffer = buffer;
+ mDestination = destination;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/DummyAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/DummyAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/DummyAuditProvider.java
index ecf25b1..777f740 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/DummyAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/DummyAuditProvider.java
@@ -1,12 +1,3 @@
-package com.xasecure.audit.provider;
-
-import com.xasecure.audit.model.AuditEventBase;
-import com.xasecure.audit.model.HBaseAuditEvent;
-import com.xasecure.audit.model.HdfsAuditEvent;
-import com.xasecure.audit.model.HiveAuditEvent;
-import com.xasecure.audit.model.KnoxAuditEvent;
-import com.xasecure.audit.model.StormAuditEvent;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -24,6 +15,10 @@ import com.xasecure.audit.model.StormAuditEvent;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package com.xasecure.audit.provider;
+
+import com.xasecure.audit.model.AuditEventBase;
+
public class DummyAuditProvider implements AuditProvider {
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
new file mode 100644
index 0000000..9acae11
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.xasecure.audit.provider;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.TreeSet;
+
+import org.apache.log4j.helpers.LogLog;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+
+public class LocalFileLogBuffer<T> implements LogBuffer<T> {
+ private String mDirectory = null;
+ private String mFile = null;
+ private String mEncoding = null;
+ private boolean mIsAppend = true;
+ private int mRolloverIntervalSeconds = 600;
+ private String mArchiveDirectory = null;
+ private int mArchiveFileCount = 10;
+
+ private Writer mWriter = null;
+ private String mCurrentFilename = null;
+ private long mNextRolloverTime = 0;
+
+ private Gson mGsonBuilder = null;
+
+ private DestinationDispatcherThread<T> mDispatcherThread = null;
+
+ public LocalFileLogBuffer() {
+ mGsonBuilder = new GsonBuilder().setPrettyPrinting().create();
+ }
+
+ public String getDirectory() {
+ return mDirectory;
+ }
+
+ public void setDirectory(String directory) {
+ mDirectory = directory;
+ }
+
+ public String getFile() {
+ return mFile;
+ }
+
+ public void setFile(String file) {
+ mFile = file;
+ }
+
+ public String getEncoding() {
+ return mEncoding;
+ }
+
+ public void setEncoding(String encoding) {
+ mEncoding = encoding;
+ }
+
+ public boolean getIsAppend() {
+ return mIsAppend;
+ }
+
+ public void setIsAppend(boolean isAppend) {
+ mIsAppend = isAppend;
+ }
+
+ public int getRolloverIntervalSeconds() {
+ return mRolloverIntervalSeconds;
+ }
+
+ public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
+ mRolloverIntervalSeconds = rolloverIntervalSeconds;
+ }
+
+ public String getArchiveDirectory() {
+ return mArchiveDirectory;
+ }
+
+ public void setArchiveDirectory(String archiveDirectory) {
+ mArchiveDirectory = archiveDirectory;
+ }
+
+ public int getArchiveFileCount() {
+ return mArchiveFileCount;
+ }
+
+ public void setArchiveFileCount(int archiveFileCount) {
+ mArchiveFileCount = archiveFileCount;
+ }
+
+
+ @Override
+ public void start(LogDestination<T> destination) {
+ LogLog.debug("==> LocalFileLogBuffer.start()");
+
+ mDispatcherThread = new DestinationDispatcherThread<T>(this, destination);
+
+ mDispatcherThread.start();
+
+ LogLog.debug("<== LocalFileLogBuffer.start()");
+ }
+
+ @Override
+ public void stop() {
+ LogLog.debug("==> LocalFileLogBuffer.stop()");
+
+ DestinationDispatcherThread<T> dispatcherThread = mDispatcherThread;
+ mDispatcherThread = null;
+
+ if(dispatcherThread != null && dispatcherThread.isAlive()) {
+ dispatcherThread.stopThread();
+
+ try {
+ dispatcherThread.join();
+ } catch (InterruptedException e) {
+ LogLog.warn("LocalFileLogBuffer.stop(): failed in waiting for DispatcherThread", e);
+ }
+ }
+
+ closeFile();
+
+ LogLog.debug("<== LocalFileLogBuffer.stop()");
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return mWriter != null;
+ }
+
+ @Override
+ public synchronized boolean add(T log) {
+ boolean ret = false;
+
+ long now = System.currentTimeMillis();
+
+ if(now > mNextRolloverTime) {
+ rollover();
+ }
+
+ Writer writer = mWriter;
+
+ if(writer != null) {
+ try {
+ String msg = toJson(log);
+
+ if(msg.contains(MiscUtil.LINE_SEPARATOR)) {
+ msg = msg.replace(MiscUtil.LINE_SEPARATOR, MiscUtil.ESCAPE_STR + MiscUtil.LINE_SEPARATOR);
+ }
+
+ writer.write(msg + MiscUtil.LINE_SEPARATOR);
+
+ ret = true;
+ } catch(IOException excp) {
+ LogLog.warn("LocalFileLogBuffer.add(): write failed", excp);
+ }
+ } else {
+ LogLog.warn("LocalFileLogBuffer.add(): writer is null");
+ }
+
+ return ret;
+ }
+
+ private synchronized void openFile() {
+ LogLog.debug("==> LocalFileLogBuffer.openFile()");
+
+ closeFile();
+
+ mCurrentFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+
+ FileOutputStream ostream = null;
+ try {
+ ostream = new FileOutputStream(mCurrentFilename, mIsAppend);
+ } catch(Exception excp) {
+ MiscUtil.createParents(new File(mCurrentFilename));
+
+ try {
+ ostream = new FileOutputStream(mCurrentFilename, mIsAppend);
+ } catch(Exception ex) {
+ // ignore; error printed down
+ }
+ }
+
+ mWriter = createWriter(ostream);
+
+ if(mWriter != null) {
+ LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mCurrentFilename);
+
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ } else {
+ LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mCurrentFilename);
+
+ mCurrentFilename = null;
+ }
+
+ LogLog.debug("<== LocalFileLogBuffer.openFile()");
+ }
+
+ private synchronized void closeFile() {
+ LogLog.debug("==> LocalFileLogBuffer.closeFile()");
+
+ Writer writer = mWriter;
+
+ mWriter = null;
+
+ if(writer != null) {
+ try {
+ writer.flush();
+ writer.close();
+ } catch(IOException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to close file " + mCurrentFilename, excp);
+ }
+
+ if(mDispatcherThread != null) {
+ mDispatcherThread.addLogfile(mCurrentFilename);
+ }
+ }
+
+ LogLog.debug("<== LocalFileLogBuffer.closeFile()");
+ }
+
+ private void rollover() {
+ LogLog.debug("==> LocalFileLogBuffer.rollover()");
+
+ closeFile();
+
+ openFile();
+
+ LogLog.debug("<== LocalFileLogBuffer.rollover()");
+ }
+
+ public OutputStreamWriter createWriter(OutputStream os ) {
+ OutputStreamWriter writer = null;
+
+ if(os != null) {
+ if(mEncoding != null) {
+ try {
+ writer = new OutputStreamWriter(os, mEncoding);
+ } catch(UnsupportedEncodingException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to create output writer.", excp);
+ }
+ }
+
+ if(writer == null) {
+ writer = new OutputStreamWriter(os);
+ }
+ }
+
+ return writer;
+ }
+
+ boolean isCurrentFilename(String filename) {
+ return mCurrentFilename != null && filename != null && filename.equals(mCurrentFilename);
+ }
+
+ private String toJson(T log) {
+ String jsonString = mGsonBuilder.toJson(log) ;
+
+ return jsonString;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("LocalFileLogBuffer {");
+ sb.append("Directory=").append(mDirectory).append("; ");
+ sb.append("File=").append(mFile).append("; ");
+ sb.append("RolloverIntervaSeconds=").append(mRolloverIntervalSeconds).append("; ");
+ sb.append("ArchiveDirectory=").append(mArchiveDirectory).append("; ");
+ sb.append("ArchiveFileCount=").append(mArchiveFileCount);
+ sb.append("}");
+
+ return sb.toString();
+ }
+
+}
+
+class DestinationDispatcherThread<T> extends Thread {
+ private TreeSet<String> mCompletedLogfiles = new TreeSet<String>();
+ private boolean mStopThread = false;
+ private LocalFileLogBuffer<T> mFileLogBuffer = null;
+ private LogDestination<T> mDestination = null;
+
+ private String mCurrentLogfile = null;
+ private BufferedReader mReader = null;
+
+ public DestinationDispatcherThread(LocalFileLogBuffer<T> fileLogBuffer, LogDestination<T> destination) {
+ super(DestinationDispatcherThread.class.getSimpleName() + "-" + System.currentTimeMillis());
+
+ mFileLogBuffer = fileLogBuffer;
+ mDestination = destination;
+
+ setDaemon(true);
+ }
+
+ public void addLogfile(String filename) {
+ LogLog.debug("==> DestinationDispatcherThread.addLogfile(" + filename + ")");
+
+ if(filename != null) {
+ synchronized(mCompletedLogfiles) {
+ mCompletedLogfiles.add(filename);
+ mCompletedLogfiles.notify();
+ }
+ }
+
+ LogLog.debug("<== DestinationDispatcherThread.addLogfile(" + filename + ")");
+ }
+
+ public void stopThread() {
+ mStopThread = true;
+ }
+
+ @Override
+ public void run() {
+ init();
+
+ // destination start() should be from the dispatcher thread
+ mDestination.start();
+
+ int pollIntervalInMs = 1000;
+
+ while(! mStopThread) {
+ synchronized(mCompletedLogfiles) {
+ while(mCompletedLogfiles.isEmpty() && !mStopThread) {
+ try {
+ mCompletedLogfiles.wait(pollIntervalInMs);
+ } catch(InterruptedException excp) {
+ LogLog.warn("LocalFileLogBuffer.run(): failed to wait for log file", excp);
+ }
+ }
+
+ mCurrentLogfile = mCompletedLogfiles.pollFirst();
+ }
+
+ if(mCurrentLogfile != null) {
+ sendCurrentFile();
+ }
+ }
+
+ mDestination.stop();
+ }
+
+ private void init() {
+ LogLog.debug("==> DestinationDispatcherThread.init()");
+
+ String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory());
+ File directory = new File(dirName);
+
+ if(directory.exists() && directory.isDirectory()) {
+ File[] files = directory.listFiles();
+
+ if(files != null) {
+ for(File file : files) {
+ if(file.exists() && file.canRead()) {
+ String filename = file.getAbsolutePath();
+ if(! mFileLogBuffer.isCurrentFilename(filename)) {
+ addLogfile(filename);
+ }
+ }
+ }
+ }
+ }
+
+ LogLog.debug("<== DestinationDispatcherThread.init()");
+ }
+
+ private boolean sendCurrentFile() {
+ boolean ret = false;
+
+ int destinationPollIntervalInMs = 1000;
+
+ openCurrentFile();
+
+ while(!mStopThread) {
+ String log = getNextStringifiedLog();
+
+ if(log == null) { // reached end-of-file
+ ret = true;
+
+ break;
+ }
+
+ // loop until log is sent successfully
+ while(!mStopThread && !mDestination.sendStringified(log)) {
+ sleep(destinationPollIntervalInMs, "LocalFileLogBuffer.sendCurrentFile(" + mCurrentLogfile + "): failed to wait for destination to be available");
+ }
+ }
+
+ closeCurrentFile();
+
+ return ret;
+ }
+
+ private String getNextStringifiedLog() {
+ String log = null;
+
+ if(mReader != null) {
+ try {
+ while(true) {
+ String line = mReader.readLine();
+
+ if(line == null) {
+ break;
+ } else {
+ if(log == null) {
+ log = "";
+ }
+
+ if(line.endsWith(MiscUtil.ESCAPE_STR)) {
+ line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
+
+ log += MiscUtil.LINE_SEPARATOR;
+ log += line;
+
+ continue;
+ } else {
+ log += line;
+ break;
+ }
+ }
+ }
+ } catch (IOException excp) {
+ LogLog.warn("getNextStringifiedLog.getNextLog(): failed to read from file " + mCurrentLogfile, excp);
+ }
+
+ if(log == null) {
+ closeCurrentFile();
+ }
+ }
+ LogLog.warn("READ: " + log);
+
+ return log;
+ }
+
+ private void openCurrentFile() {
+ LogLog.debug("==> openCurrentFile(" + mCurrentLogfile + ")");
+
+ closeCurrentFile();
+
+ while(mReader == null) {
+ if(mCurrentLogfile != null) {
+ try {
+ FileInputStream inStr = new FileInputStream(mCurrentLogfile);
+
+ InputStreamReader strReader = createReader(inStr);
+
+ if(strReader != null) {
+ mReader = new BufferedReader(strReader);
+ }
+ } catch(FileNotFoundException excp) {
+ LogLog.warn("openNextFile(): error while opening file " + mCurrentLogfile, excp);
+ }
+ }
+ }
+
+ LogLog.debug("<== openCurrentFile(" + mCurrentLogfile + ")");
+ }
+
+ private void closeCurrentFile() {
+ LogLog.debug("==> closeCurrentFile(" + mCurrentLogfile + ")");
+
+ if(mReader != null) {
+ try {
+ mReader.close();
+ } catch(IOException excp) {
+ // ignore
+ }
+ }
+ mReader = null;
+
+ if(!mStopThread) {
+ archiveCurrentFile();
+ }
+
+ LogLog.debug("<== closeCurrentFile(" + mCurrentLogfile + ")");
+ }
+
+ private void archiveCurrentFile() {
+ if(mCurrentLogfile != null) {
+ File logFile = new File(mCurrentLogfile);
+ String archiveDirName = MiscUtil.replaceTokens(mFileLogBuffer.getArchiveDirectory());
+ String archiveFilename = archiveDirName + File.separator + MiscUtil.replaceTokens(logFile.getName());
+
+ try {
+ if(logFile.exists()) {
+ File archiveFile = new File(archiveFilename);
+
+ MiscUtil.createParents(archiveFile);
+
+ if(! logFile.renameTo(archiveFile)) {
+ // TODO: renameTo() does not work in all cases. in case of failure, copy the file contents to the destination and delete the file
+ }
+
+ File archiveDir = new File(archiveDirName);
+ File[] files = archiveDir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File f) {
+ return f.isFile();
+ }
+ });
+
+ int numOfFilesToDelete = files == null ? 0 : (files.length - mFileLogBuffer.getArchiveFileCount());
+
+ if(numOfFilesToDelete > 0) {
+ Arrays.sort(files, new Comparator<File>() {
+
+ @Override
+ public int compare(File f1, File f2) {
+ return (int)(f1.lastModified() - f2.lastModified());
+ }
+ });
+
+ for(int i = 0; i < numOfFilesToDelete; i++) {
+ LogLog.debug("DELETE: " + files[i].getAbsolutePath());
+
+ files[i].delete();
+ }
+ }
+ }
+ } catch(Exception excp) {
+ LogLog.warn("archiveCurrentFile(): faile to move " + mCurrentLogfile + " to archive location " + archiveFilename, excp);
+ }
+ }
+ mCurrentLogfile = null;
+ }
+
+ public InputStreamReader createReader(InputStream iStr) {
+ InputStreamReader reader = null;
+
+ if(iStr != null) {
+ String encoding = mFileLogBuffer.getEncoding();
+
+ if(encoding != null) {
+ try {
+ reader = new InputStreamReader(iStr, encoding);
+ } catch(UnsupportedEncodingException excp) {
+ LogLog.warn("createReader(): failed to create input reader.", excp);
+ }
+ }
+
+ if(reader == null) {
+ reader = new InputStreamReader(iStr);
+ }
+ }
+
+ return reader;
+ }
+
+ private void sleep(int sleepTimeInMs, String onFailMsg) {
+ try {
+ Thread.sleep(sleepTimeInMs);
+ } catch(InterruptedException excp) {
+ LogLog.warn(onFailMsg, excp);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("DestinationDispatcherThread {");
+ sb.append("ThreadName=").append(this.getName()).append("; ");
+ sb.append("CompletedLogfiles.size()=").append(mCompletedLogfiles.size()).append("; ");
+ sb.append("StopThread=").append(mStopThread).append("; ");
+ sb.append("CurrentLogfile=").append(mCurrentLogfile);
+ sb.append("}");
+
+ return sb.toString();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
new file mode 100644
index 0000000..ce67e01
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.xasecure.audit.provider;
+
+
+public interface LogBuffer<T> {
+ public void start(LogDestination<T> destination);
+
+ public void stop();
+
+ boolean isAvailable();
+
+ public boolean add(T log);
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/LogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LogDestination.java
new file mode 100644
index 0000000..a8b5081
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LogDestination.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.xasecure.audit.provider;
+
+
+public interface LogDestination<T> {
+ public void start();
+
+ public void stop();
+
+ boolean isAvailable();
+
+ public boolean send(T log);
+
+ public boolean sendStringified(String log);
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
new file mode 100644
index 0000000..6610210
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -0,0 +1,139 @@
+package com.xasecure.audit.provider;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.rmi.dgc.VMID;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.log4j.helpers.LogLog;
+
+public class MiscUtil {
+ public static final String TOKEN_HOSTNAME = "%hostname%";
+ public static final String TOKEN_APP_INSTANCE = "%app-instance%";
+ public static final String TOKEN_CREATE_TIME_START = "%create-time:";
+ public static final String TOKEN_CREATE_TIME_END = "%";
+ public static final String ESCAPE_STR = "\\";
+
+ static VMID sJvmID = new VMID();
+
+ public static String LINE_SEPARATOR = System.getProperty("line.separator");
+
+ public static String replaceTokens(String str) {
+ if(str == null) {
+ return str;
+ }
+
+ str = replaceHostname(str);
+ str = replaceAppInstance(str);
+ str = replaceCreateTime(str);
+
+ return str;
+ }
+
+ public static String replaceHostname(String str) {
+ if(!str.contains(TOKEN_HOSTNAME)) {
+ return str;
+ }
+
+ String hostName = null;
+
+ try {
+ hostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException excp) {
+ LogLog.warn("LocalFileLogBuffer", excp);
+ }
+
+ if(hostName == null) {
+ hostName = "Unknown";
+ }
+
+ return str.replace(TOKEN_HOSTNAME, hostName);
+ }
+
+ public static String replaceAppInstance(String str) {
+ if(!str.contains(TOKEN_APP_INSTANCE)) {
+ return str;
+ }
+
+ String appInstance = Integer.toString(Math.abs(sJvmID.hashCode()));
+
+ return str.replace(TOKEN_APP_INSTANCE, appInstance);
+ }
+
+ public static String replaceCreateTime(String str) {
+ Date now = new Date();
+
+ while(str.contains(TOKEN_CREATE_TIME_START)) {
+ int tagStartPos = str.indexOf(TOKEN_CREATE_TIME_START);
+ int tagEndPos = str.indexOf(TOKEN_CREATE_TIME_END, tagStartPos + TOKEN_CREATE_TIME_START.length());
+
+ if(tagEndPos <= tagStartPos) {
+ break;
+ }
+
+ String tag = str.substring(tagStartPos, tagEndPos+1);
+ String dtFormat = tag.substring(TOKEN_CREATE_TIME_START.length(), tag.lastIndexOf(TOKEN_CREATE_TIME_END));
+
+ String replaceStr = "";
+
+ if(dtFormat != null) {
+ SimpleDateFormat sdf = new SimpleDateFormat(dtFormat);
+
+ replaceStr = sdf.format(now);
+ }
+
+ str = str.replace(tag, replaceStr);
+ }
+
+ return str;
+ }
+
+ public static void createParents(File file) {
+ if(file != null) {
+ String parentName = file.getParent();
+
+ if (parentName != null) {
+ File parentDir = new File(parentName);
+
+ if(!parentDir.exists()) {
+ parentDir.mkdirs();
+ }
+ }
+ }
+ }
+
+ public static long getNextRolloverTime(long lastRolloverTime, long interval) {
+ long now = System.currentTimeMillis() / 1000 * 1000; // round to second
+
+ if(lastRolloverTime <= 0) {
+ // should this be set to the next multiple-of-the-interval from start of the day?
+ return now + interval;
+ } else if(lastRolloverTime <= now) {
+ long nextRolloverTime = now + interval;
+
+ // keep it at 'interval' boundary
+ long trimInterval = (nextRolloverTime - lastRolloverTime) % interval;
+
+ return nextRolloverTime - trimInterval;
+ } else {
+ return lastRolloverTime;
+ }
+ }
+
+ public static int parseInteger(String str, int defValue) {
+ int ret = defValue;
+
+ if(str != null) {
+ try {
+ ret = Integer.parseInt(str);
+ } catch(Exception excp) {
+ // ignore
+ }
+ }
+
+ return ret;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
new file mode 100644
index 0000000..db8489c
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -0,0 +1,48 @@
+package com.xasecure.audit.provider.hdfs;
+
+import java.util.Map;
+
+import com.xasecure.audit.model.AuditEventBase;
+import com.xasecure.audit.provider.BufferedAuditProvider;
+import com.xasecure.audit.provider.LocalFileLogBuffer;
+import com.xasecure.audit.provider.MiscUtil;
+
+public class HdfsAuditProvider extends BufferedAuditProvider {
+
+ public HdfsAuditProvider() {
+ }
+
+ public void init(Map<String, String> properties) {
+ String encoding = properties.get("encoding");
+
+ String hdfsDestinationDirectory = properties.get("destination.directroy");
+ String hdfsDestinationFile = properties.get("destination.file");
+ int hdfsDestinationRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.rollover.interval.seconds"), 24 * 60 * 60);
+
+ String localFileBufferDirectory = properties.get("local.buffer.directroy");
+ String localFileBufferFile = properties.get("local.buffer.file");
+ int localFileBufferRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("local.buffer.rollover.interval.seconds"), 10 * 60);
+ String localFileBufferArchiveDirectory = properties.get("local.archive.directroy");
+ int localFileBufferArchiveFileCount = MiscUtil.parseInteger(properties.get("local.archive.max.file.count"), 10);
+
+ HdfsLogDestination<AuditEventBase> mHdfsDestination = new HdfsLogDestination<AuditEventBase>();
+
+ mHdfsDestination.setDirectory(hdfsDestinationDirectory);
+ mHdfsDestination.setFile(hdfsDestinationFile);
+ mHdfsDestination.setEncoding(encoding);
+ mHdfsDestination.setRolloverIntervalSeconds(hdfsDestinationRolloverIntervalSeconds);
+
+ LocalFileLogBuffer<AuditEventBase> mLocalFileBuffer = new LocalFileLogBuffer<AuditEventBase>();
+
+ mLocalFileBuffer.setDirectory(localFileBufferDirectory);
+ mLocalFileBuffer.setFile(localFileBufferFile);
+ mLocalFileBuffer.setEncoding(encoding);
+ mLocalFileBuffer.setRolloverIntervalSeconds(localFileBufferRolloverIntervalSeconds);
+ mLocalFileBuffer.setArchiveDirectory(localFileBufferArchiveDirectory);
+ mLocalFileBuffer.setArchiveFileCount(localFileBufferArchiveFileCount);
+
+ setBufferAndDestination(mLocalFileBuffer, mHdfsDestination);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
new file mode 100644
index 0000000..eeb7574
--- /dev/null
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.xasecure.audit.provider.hdfs;
+
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.helpers.LogLog;
+
+import com.xasecure.audit.provider.LogDestination;
+import com.xasecure.audit.provider.MiscUtil;
+
+public class HdfsLogDestination<T> implements LogDestination<T> {
+ private String mDirectory = null;
+ private String mFile = null;
+ private String mEncoding = null;
+ private boolean mIsAppend = true;
+ private int mRolloverIntervalSeconds = 24 * 60 * 60;
+
+ private OutputStreamWriter mWriter = null;
+ private String mCurrentFilename = null;
+ private long mNextRolloverTime = 0;
+ private boolean mIsStopInProgress = false;
+
+ public HdfsLogDestination() {
+ }
+
+ public String getDirectory() {
+ return mDirectory;
+ }
+
+ public void setDirectory(String directory) {
+ this.mDirectory = directory;
+ }
+
+ public String getFile() {
+ return mFile;
+ }
+
+ public void setFile(String file) {
+ this.mFile = file;
+ }
+
+ public String getEncoding() {
+ return mEncoding;
+ }
+
+ public void setEncoding(String encoding) {
+ mEncoding = encoding;
+ }
+
+ public int getRolloverIntervalSeconds() {
+ return mRolloverIntervalSeconds;
+ }
+
+ public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
+ this.mRolloverIntervalSeconds = rolloverIntervalSeconds;
+ }
+
+ @Override
+ public void start() {
+ LogLog.debug("==> HdfsLogDestination.start()");
+
+ openFile();
+
+ LogLog.debug("<== HdfsLogDestination.start()");
+ }
+
+ @Override
+ public void stop() {
+ LogLog.debug("==> HdfsLogDestination.stop()");
+
+ mIsStopInProgress = true;
+
+ closeFile();
+
+ mIsStopInProgress = false;
+
+ LogLog.debug("<== HdfsLogDestination.stop()");
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return mWriter != null;
+ }
+
+ @Override
+ public boolean send(T log) {
+ boolean ret = false;
+
+ if(log != null) {
+ String msg = log.toString();
+
+ ret = sendStringified(msg);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public boolean sendStringified(String log) {
+ boolean ret = false;
+
+ rolloverIfNeeded();
+
+ OutputStreamWriter writer = mWriter;
+
+ if(writer != null) {
+ try {
+ writer.write(log);
+
+ ret = true;
+ } catch (IOException excp) {
+ LogLog.warn("HdfsLogDestination.sendStringified(): write failed", excp);
+ }
+ }
+
+ return ret;
+ }
+
+ private void openFile() {
+ LogLog.debug("==> HdfsLogDestination.openFile()");
+
+ closeFile();
+
+ mCurrentFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile);
+
+ FSDataOutputStream ostream = null;
+ FileSystem fileSystem = null;
+ Path pathLogfile = null;
+ Configuration conf = null;
+
+ try {
+ LogLog.debug("HdfsLogDestination.openFile(): opening file " + mCurrentFilename);
+
+ URI uri = URI.create(mCurrentFilename);
+
+ // TODO: mechanism to XA-HDFS plugin to disable auditing of access checks to the current HDFS file
+
+ conf = new Configuration();
+ pathLogfile = new Path(mCurrentFilename);
+ fileSystem = FileSystem.get(uri, conf);
+
+ if(fileSystem.exists(pathLogfile)) {
+ if(mIsAppend) {
+ try {
+ ostream = fileSystem.append(pathLogfile);
+ } catch(IOException excp) {
+ // append may not be supported by the filesystem. rename existing file and create a new one
+ String fileSuffix = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_CREATE_TIME_START + "yyyyMMdd-HHmm.ss" + MiscUtil.TOKEN_CREATE_TIME_END);
+ String movedFilename = appendToFilename(mCurrentFilename, fileSuffix);
+ Path movedFilePath = new Path(movedFilename);
+
+ fileSystem.rename(pathLogfile, movedFilePath);
+ }
+ }
+ }
+
+ if(ostream == null){
+ ostream = fileSystem.create(pathLogfile);
+ }
+ } catch(IOException ex) {
+ Path parentPath = pathLogfile.getParent();
+
+ try {
+ if(parentPath != null&& fileSystem != null && !fileSystem.exists(parentPath) && fileSystem.mkdirs(parentPath)) {
+ ostream = fileSystem.create(pathLogfile);
+ }
+ } catch (IOException e) {
+ LogLog.warn("HdfsLogDestination.openFile() failed", e);
+ } catch (Throwable e) {
+ LogLog.warn("HdfsLogDestination.openFile() failed", e);
+ }
+ } catch(Throwable ex) {
+ LogLog.warn("HdfsLogDestination.openFile() failed", ex);
+ } finally {
+ // TODO: unset the property set above to exclude auditing of logfile opening
+ // System.setProperty(hdfsCurrentFilenameProperty, null);
+ }
+
+ mWriter = createWriter(ostream);
+
+ if(mWriter != null) {
+ LogLog.debug("HdfsLogDestination.openFile(): opened file " + mCurrentFilename);
+
+ mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000));
+ } else {
+ LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mCurrentFilename);
+
+ mCurrentFilename = null;
+ }
+
+ LogLog.debug("<== HdfsLogDestination.openFile(" + mCurrentFilename + ")");
+ }
+
+ private void closeFile() {
+ LogLog.debug("==> HdfsLogDestination.closeFile()");
+
+ OutputStreamWriter writer = mWriter;
+
+ mWriter = null;
+
+ if(writer != null) {
+ try {
+ writer.flush();
+ writer.close();
+ } catch(IOException excp) {
+ if(! mIsStopInProgress) { // during shutdown, the underlying FileSystem might already be closed; so don't print error details
+ LogLog.warn("HdfsLogDestination: failed to close file " + mCurrentFilename, excp);
+ }
+ }
+ }
+
+ LogLog.debug("<== HdfsLogDestination.closeFile()");
+ }
+
+ private void rollover() {
+ LogLog.debug("==> HdfsLogDestination.rollover()");
+
+ closeFile();
+
+ openFile();
+
+ LogLog.debug("<== HdfsLogDestination.rollover()");
+ }
+
+ private void rolloverIfNeeded() {
+ long now = System.currentTimeMillis();
+
+ if(now > mNextRolloverTime) {
+ rollover();
+ }
+ }
+
+ private OutputStreamWriter createWriter(OutputStream os ) {
+ OutputStreamWriter writer = null;
+
+ if(os != null) {
+ if(mEncoding != null) {
+ try {
+ writer = new OutputStreamWriter(os, mEncoding);
+ } catch(UnsupportedEncodingException excp) {
+ LogLog.warn("LocalFileLogBuffer: failed to create output writer.", excp);
+ }
+ }
+
+ if(writer == null) {
+ writer = new OutputStreamWriter(os);
+ }
+ }
+
+ return writer;
+ }
+
+ private String appendToFilename(String fileName, String strToAppend) {
+ String ret = fileName;
+
+ if(strToAppend != null) {
+ if(ret == null) {
+ ret = "";
+ }
+
+ int extnPos = ret.lastIndexOf(".");
+
+ if(extnPos < 0) {
+ ret += strToAppend;
+ } else {
+ String extn = ret.substring(extnPos);
+
+ ret = ret.substring(0, extnPos) + strToAppend + extn;
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("HdfsLogDestination {");
+ sb.append("Directory=").append(mDirectory).append("; ");
+ sb.append("File=").append(mFile).append("; ");
+ sb.append("RolloverIntervalSeconds=").append(mRolloverIntervalSeconds);
+ sb.append("}");
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/hbase-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit.xml b/hbase-agent/conf/xasecure-audit.xml
index be1b900..f33b8ba 100644
--- a/hbase-agent/conf/xasecure-audit.xml
+++ b/hbase-agent/conf/xasecure-audit.xml
@@ -87,4 +87,76 @@
<name>xasecure.audit.db.batch.size</name>
<value>100</value>
</property>
+
+
+ <!-- HDFS audit provider configuration -->
+ <property>
+ <name>xasecure.audit.hdfs.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.resume.queue.size</name>
+ <value>8192</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.encoding</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.directroy</name>
+ <value>hdfs://namenodehost:8020/audit/hbase/%create-time:yyyyMMdd%</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.file</name>
+ <value>%hostname%-audit.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
+ <value>/tmp/logs/hbase</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file</name>
+ <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.directroy</name>
+ <value>/tmp/logs/archive/hbase</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name>
+ <value>10</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/hdfs-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit.xml b/hdfs-agent/conf/xasecure-audit.xml
index 2b24f33..1ae6f3b 100644
--- a/hdfs-agent/conf/xasecure-audit.xml
+++ b/hdfs-agent/conf/xasecure-audit.xml
@@ -1,13 +1,49 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ </property>
<property>
- <name>xasecure.audit.provider.factory</name>
- <value>com.xasecure.audit.provider.AuditProviderFactory</value>
- </property>
+ <name>xasecure.audit.repository.name</name>
+ <value>hadoopdev</value>
+ </property>
+
+
+ <!-- DB audit provider configuration -->
+ <property>
+ <name>xasecure.audit.db.is.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.async.resume.queue.size</name>
+ <value>8192</value>
+ </property>
- <!-- Properties whose name begin with "xasecure.audit." are used to configure JPA -->
+ <property>
+ <name>xasecure.audit.db.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.batch.size</name>
+ <value>100</value>
+ </property>
+
+ <!-- Properties whose name begin with "xasecure.audit.jpa." are used to configure JPA -->
<property>
<name>xasecure.audit.jpa.javax.persistence.jdbc.url</name>
<value>jdbc:mysql://localhost:3306/xa_db</value>
@@ -32,60 +68,103 @@
<name>xasecure.audit.credential.provider.file</name>
<value>jceks://file/etc/xasecure/conf/auditcred.jceks</value>
</property>
-
- <property>
- <name>xasecure.audit.repository.name</name>
- <value>hadoopdev</value>
- </property>
-
- <property>
- <name>xasecure.audit.is.enabled</name>
- <value>true</value>
- </property>
+
+ <!-- HDFS audit provider configuration -->
<property>
- <name>xasecure.audit.log4j.is.enabled</name>
+ <name>xasecure.audit.hdfs.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.log4j.is.async</name>
+ <name>xasecure.audit.hdfs.is.async</name>
<value>false</value>
</property>
-
+
<property>
- <name>xasecure.audit.log4j.async.max.queue.size</name>
+ <name>xasecure.audit.hdfs.async.max.queue.size</name>
<value>10240</value>
</property>
<property>
- <name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.hdfs.async.resume.queue.size</name>
+ <value>8192</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
-
+
<property>
- <name>xasecure.audit.db.is.enabled</name>
- <value>true</value>
+ <name>xasecure.audit.hdfs.config.encoding</name>
+ <value></value>
</property>
-
+
<property>
- <name>xasecure.audit.db.is.async</name>
+ <name>xasecure.audit.hdfs.config.destination.directroy</name>
+ <value>hdfs://namenodehost:8020/audit/hdfs/%create-time:yyyyMMdd%</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.file</name>
+ <value>%hostname%-audit.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
+ <value>/tmp/logs/hdfs</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file</name>
+ <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.directroy</name>
+ <value>/tmp/logs/archive/hdfs</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name>
+ <value>10</value>
+ </property>
+
+ <!-- Log4j audit provider configuration -->
+ <property>
+ <name>xasecure.audit.log4j.is.enabled</name>
<value>false</value>
</property>
-
+
<property>
- <name>xasecure.audit.db.async.max.queue.size</name>
+ <name>xasecure.audit.log4j.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.log4j.async.max.queue.size</name>
<value>10240</value>
</property>
<property>
- <name>xasecure.audit.db.async.max.flush.interval.ms</name>
- <value>30000</value>
+ <name>xasecure.audit.log4j.async.resume.queue.size</name>
+ <value>8192</value>
</property>
<property>
- <name>xasecure.audit.db.batch.size</name>
- <value>100</value>
+ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
+ <value>30000</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/hive-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit.xml b/hive-agent/conf/xasecure-audit.xml
index 4014546..eb951a4 100644
--- a/hive-agent/conf/xasecure-audit.xml
+++ b/hive-agent/conf/xasecure-audit.xml
@@ -87,4 +87,76 @@
<name>xasecure.audit.db.batch.size</name>
<value>100</value>
</property>
+
+
+ <!-- HDFS audit provider configuration -->
+ <property>
+ <name>xasecure.audit.hdfs.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.resume.queue.size</name>
+ <value>8192</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.encoding</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.directroy</name>
+ <value>hdfs://namenodehost:8020/audit/hive/%create-time:yyyyMMdd%</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.file</name>
+ <value>%hostname%-audit.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
+ <value>/tmp/logs/hive</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file</name>
+ <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.directroy</name>
+ <value>/tmp/logs/archive/hive</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name>
+ <value>10</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/knox-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit.xml b/knox-agent/conf/xasecure-audit.xml
index a5252d1..987a49d 100644
--- a/knox-agent/conf/xasecure-audit.xml
+++ b/knox-agent/conf/xasecure-audit.xml
@@ -82,4 +82,76 @@
<name>xasecure.audit.db.batch.size</name>
<value>100</value>
</property>
+
+
+ <!-- HDFS audit provider configuration -->
+ <property>
+ <name>xasecure.audit.hdfs.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.resume.queue.size</name>
+ <value>8192</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.encoding</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.directroy</name>
+ <value>hdfs://namenodehost:8020/audit/knox/%create-time:yyyyMMdd%</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.file</name>
+ <value>%hostname%-audit.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
+ <value>/tmp/logs/knox</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file</name>
+ <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.directroy</name>
+ <value>/tmp/logs/archive/knox</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name>
+ <value>10</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/5ccf382d/storm-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit.xml b/storm-agent/conf/xasecure-audit.xml
index 4014546..ef0b27a 100644
--- a/storm-agent/conf/xasecure-audit.xml
+++ b/storm-agent/conf/xasecure-audit.xml
@@ -87,4 +87,76 @@
<name>xasecure.audit.db.batch.size</name>
<value>100</value>
</property>
+
+
+ <!-- HDFS audit provider configuration -->
+ <property>
+ <name>xasecure.audit.hdfs.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.resume.queue.size</name>
+ <value>8192</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.encoding</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.directroy</name>
+ <value>hdfs://namenodehost:8020/audit/storm/%create-time:yyyyMMdd%</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.file</name>
+ <value>%hostname%-audit.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.directroy</name>
+ <value>/tmp/logs/storm</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file</name>
+ <value>%create-time:yyyyMMdd-HHmm.ss%.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.directroy</name>
+ <value>/tmp/logs/archive/storm</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name>
+ <value>10</value>
+ </property>
</configuration>
[04/10] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-argus
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-argus
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/8bafd82d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/8bafd82d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/8bafd82d
Branch: refs/heads/master
Commit: 8bafd82db15554c9d8f8c1756c26f16cd10b136f
Parents: 2fde332 bb161f3
Author: mneethiraj <mn...@hortonworks.com>
Authored: Sat Sep 20 01:49:24 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Sat Sep 20 01:49:24 2014 -0700
----------------------------------------------------------------------
hbase-agent/scripts/install.sh | 8 ++++----
hdfs-agent/conf/xasecure-hadoop-env.sh | 2 +-
hdfs-agent/scripts/install.sh | 6 +++---
hive-agent/scripts/install.sh | 8 ++++----
4 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[05/10] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-argus.git
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-argus.git
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/886e9c24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/886e9c24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/886e9c24
Branch: refs/heads/master
Commit: 886e9c241e3919ff2a17767a4db3e1328ed7d655
Parents: 8bafd82 1e1fcff
Author: mneethiraj <mn...@hortonworks.com>
Authored: Sat Sep 20 23:58:07 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Sat Sep 20 23:58:07 2014 -0700
----------------------------------------------------------------------
knox-agent/scripts/install.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[02/10] git commit: - misc updates
Posted by ma...@apache.org.
- misc updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/f7c934cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/f7c934cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/f7c934cf
Branch: refs/heads/master
Commit: f7c934cf47ca566530badd821a54eaf4481db197
Parents: be47e64
Author: mneethiraj <mn...@hortonworks.com>
Authored: Fri Sep 19 00:09:43 2014 -0700
Committer: mneethiraj <mn...@hortonworks.com>
Committed: Fri Sep 19 00:09:43 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/log4j/HdfsLogDestination.java | 2 +-
.../src/main/java/org/apache/log4j/LocalFileLogBuffer.java | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/f7c934cf/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java b/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
index ec53977..27def91 100644
--- a/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
+++ b/agents-common/src/main/java/org/apache/log4j/HdfsLogDestination.java
@@ -119,7 +119,6 @@ public class HdfsLogDestination implements LogDestination<String> {
if(writer != null) {
try {
writer.write(log + LINE_SEPARATOR);
- writer.flush();
ret = true;
} catch (IOException excp) {
@@ -217,6 +216,7 @@ public class HdfsLogDestination implements LogDestination<String> {
if(writer != null) {
try {
+ writer.flush();
writer.close();
} catch(IOException excp) {
LogLog.warn("HdfsLogDestination: failed to close file " + mCurrentFilename, excp);
http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/f7c934cf/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java b/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
index f52f042..df50f8d 100644
--- a/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
+++ b/agents-common/src/main/java/org/apache/log4j/LocalFileLogBuffer.java
@@ -207,7 +207,7 @@ public class LocalFileLogBuffer implements LogBuffer<String> {
updateNextRolloverTime();
} else {
- LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write" + mCurrentFilename);
+ LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mCurrentFilename);
mCurrentFilename = null;
}
@@ -224,6 +224,7 @@ public class LocalFileLogBuffer implements LogBuffer<String> {
if(writer != null) {
try {
+ writer.flush();
writer.close();
} catch(IOException excp) {
LogLog.warn("LocalFileLogBuffer: failed to close file " + mCurrentFilename, excp);