You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/12/28 13:55:27 UTC
incubator-impala git commit: IMPALA-4431: Add audit event log control
mechanism to prevent disk overflow
Repository: incubator-impala
Updated Branches:
refs/heads/master 5eef1bf82 -> 218ed9215
IMPALA-4431: Add audit event log control mechanism to prevent disk
overflow
There is no limit to the number of audit event log files. When audit
event log is enabled the growing number of log files will fill up the
disk space.
This patch adds checking and rotation mechanism on audit event log
files to prevent file number out of control which causes disk overflow.
Change-Id: I8c3229cbdb6275f969c15258c9ccab6efeb24368
Reviewed-on: http://gerrit.cloudera.org:8080/4971
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/218ed921
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/218ed921
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/218ed921
Branch: refs/heads/master
Commit: 218ed9215648427dc08ece675e2a90088b71aa6c
Parents: 5eef1bf
Author: davidxdh <da...@163.com>
Authored: Sun Nov 6 16:27:32 2016 +0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Dec 28 05:25:38 2016 +0000
----------------------------------------------------------------------
be/src/common/init.cc | 6 ++++++
be/src/common/logging.cc | 16 ++++++++++++++++
be/src/common/logging.h | 9 +++++++--
be/src/service/impala-server.cc | 2 +-
be/src/service/impala-server.h | 3 +++
5 files changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 210a642..a1e734b 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -65,6 +65,10 @@ DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per sever
"level. The most recent log files are retained. If set to 0, all log files are "
"retained.");
+DEFINE_int32(max_audit_event_log_files, 0, "Maximum number of audit event log files "
+ "to retain. The most recent audit event log files are retained. If set to 0, "
+ "all audit event log files are retained.");
+
DEFINE_int64(pause_monitor_sleep_time_ms, 500, "Sleep time in milliseconds for "
"pause monitor thread.");
@@ -142,6 +146,8 @@ static scoped_ptr<impala::Thread> pause_monitor;
// Check for log rotation in every interval of the maintenance thread
impala::CheckAndRotateLogFiles(FLAGS_max_log_files);
+ // Check for audit event log rotation in every interval of the maintenance thread
+ impala::CheckAndRotateAuditEventLogFiles(FLAGS_max_audit_event_log_files);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/common/logging.cc
----------------------------------------------------------------------
diff --git a/be/src/common/logging.cc b/be/src/common/logging.cc
index a8430c3..4ef1aeb 100644
--- a/be/src/common/logging.cc
+++ b/be/src/common/logging.cc
@@ -32,6 +32,7 @@
#include <stdio.h>
#include "common/logging.h"
+#include "service/impala-server.h"
#include "util/error-util.h"
#include "util/logging-support.h"
#include "util/redactor.h"
@@ -47,6 +48,8 @@ DEFINE_bool(redirect_stdout_stderr, true,
DECLARE_string(redaction_rules_file);
+DECLARE_string(audit_event_log_dir);
+
using boost::uuids::random_generator;
bool logging_initialized = false;
@@ -170,3 +173,16 @@ void impala::CheckAndRotateLogFiles(int max_log_files) {
impala::LoggingSupport::DeleteOldLogs(fname, max_log_files);
}
}
+
+void impala::CheckAndRotateAuditEventLogFiles(int max_log_files) {
+ // Return if audit event logging is disabled
+ if (FLAGS_audit_event_log_dir.empty()) return;
+ // Ignore bad input or disable log rotation
+ if (max_log_files <= 0) return;
+ // Check audit event log files
+ // Build glob pattern for input e.g. /tmp/impala_audit_event_log_1.0-*
+ string fname = strings::Substitute(
+ "$0/$1*", FLAGS_audit_event_log_dir, ImpalaServer::AUDIT_EVENT_LOG_FILE_PREFIX);
+
+ impala::LoggingSupport::DeleteOldLogs(fname, max_log_files);
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/common/logging.h
----------------------------------------------------------------------
diff --git a/be/src/common/logging.h b/be/src/common/logging.h
index 03a8294..057954b 100644
--- a/be/src/common/logging.h
+++ b/be/src/common/logging.h
@@ -82,9 +82,14 @@ void ShutdownLogging();
/// Writes all command-line flags to the log at level INFO.
void LogCommandLineFlags();
-/// Helper function that checks for the number of logfiles in the log directory and removes
-/// the oldest ones given an upper bound of number of logfiles to keep.
+/// Helper function that checks for the number of logfiles in the log directory and
+/// removes the oldest ones given an upper bound of number of logfiles to keep.
void CheckAndRotateLogFiles(int max_log_files);
+
+/// Helper function that checks for the number of audit event logfiles in the log
+/// directory and removes the oldest ones given an upper bound of number of audit event
+/// logfiles to keep.
+void CheckAndRotateAuditEventLogFiles(int max_log_files);
}
#endif // IR_COMPILE
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index a5f176b..e159fa2 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -193,7 +193,7 @@ namespace impala {
// relative to UTC. The same time zone change was made for the audit log, but the
// version was kept at 1.0 because there is no known consumer of the timestamp.
const string PROFILE_LOG_FILE_PREFIX = "impala_profile_log_1.1-";
-const string AUDIT_EVENT_LOG_FILE_PREFIX = "impala_audit_event_log_1.0-";
+const string ImpalaServer::AUDIT_EVENT_LOG_FILE_PREFIX = "impala_audit_event_log_1.0-";
const string LINEAGE_LOG_FILE_PREFIX = "impala_lineage_log_1.0-";
const uint32_t MAX_CANCELLATION_QUEUE_SIZE = 65536;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/218ed921/be/src/service/impala-server.h
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index e2f9c67..0697357 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -270,6 +270,9 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// Returns true if lineage logging is enabled, false otherwise.
bool IsLineageLoggingEnabled();
+ /// The prefix of audit event log filename.
+ static const string AUDIT_EVENT_LOG_FILE_PREFIX;
+
private:
friend class ChildQuery;
friend class ImpalaHttpHandler;