You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2018/10/10 00:31:31 UTC
kudu git commit: Improve logging for mini HMS and Sentry services
Repository: kudu
Updated Branches:
refs/heads/master 88edfc36a -> 61aed1a58
Improve logging for mini HMS and Sentry services
This commit adds custom logging configurations for both mini HMS and
Sentry. Both services now log warnings or errors to the console, and
info level messages to a file in the mini cluster root. This cuts down
on verbosity when both are active, and simplifies debugging by
separating full-fidelity logs.
The HMS console log is set to ERROR level and the Sentry console log is
set to the WARN level, based on the relative verbosity of the service
logs. Adjusting these levels as necessary in the future will be
straightforward.
Change-Id: Idd14cda83d56b0588667b9a3de260ffa62842adc
Reviewed-on: http://gerrit.cloudera.org:8080/11635
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao <ha...@cloudera.com>
Reviewed-by: Andrew Wong <aw...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/61aed1a5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/61aed1a5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/61aed1a5
Branch: refs/heads/master
Commit: 61aed1a58d1510d65cb9a3370ffbe93431e54d58
Parents: 88edfc3
Author: Dan Burkert <da...@apache.org>
Authored: Mon Oct 8 11:24:14 2018 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Wed Oct 10 00:31:17 2018 +0000
----------------------------------------------------------------------
src/kudu/hms/mini_hms.cc | 38 +++++++++++++++++++++++++++++++++----
src/kudu/hms/mini_hms.h | 3 +++
src/kudu/sentry/mini_sentry.cc | 29 ++++++++++++++++++++++++++--
3 files changed, 64 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/61aed1a5/src/kudu/hms/mini_hms.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc
index 079cdb1..771cc30 100644
--- a/src/kudu/hms/mini_hms.cc
+++ b/src/kudu/hms/mini_hms.cc
@@ -102,6 +102,7 @@ Status MiniHms::Start() {
RETURN_NOT_OK(CreateHiveSite());
RETURN_NOT_OK(CreateCoreSite());
+ RETURN_NOT_OK(CreateLogConfig());
// Comma-separated list of additional jars to add to the HMS classpath.
string aux_jars = Substitute("$0/hms-plugin.jar,$1/hcatalog/share/hcatalog/*",
@@ -111,10 +112,6 @@ Status MiniHms::Start() {
string java_options =
// Ensure IPv4 is used.
"-Djava.net.preferIPv4Stack=true "
- // Make logging less verbose.
- "-Dhive.log.level=WARN "
- // Log to the console.
- "-Dhive.root.logger=console "
// Tune down the Derby deadlock timeout. The HMS's use of Derby with the
// NOTIFICATION_SEQUENCE table is prone to deadlocks, at which point Derby
// cancels a conflicting transaction after waiting out the timeout. This
@@ -321,5 +318,38 @@ Status MiniHms::CreateCoreSite() const {
JoinPathSegments(data_root_, "core-site.xml"));
}
+Status MiniHms::CreateLogConfig() const {
+ // Configure the HMS to output ERROR messages to the stderr console, and INFO
+ // and above to hms.log in the data root. The console messages have a special
+ // 'HMS' tag included to disambiguate them from other Java component logs. The
+ // HMS automatically looks for a logging configuration named
+ // 'hive-log4j2.properties' in the configured HIVE_CONF_DIR.
+ static const string kFileTemplate = R"(
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [HMS - %p - %t] (%F:%L) %m%n
+appender.console.filter.threshold.type = ThresholdFilter
+appender.console.filter.threshold.level = ERROR
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = $0
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+rootLogger.level = INFO
+rootLogger.appenderRefs = console, file
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+ )";
+
+ string file_contents = Substitute(kFileTemplate, JoinPathSegments(data_root_, "hms.log"));
+
+ return WriteStringToFile(Env::Default(),
+ file_contents,
+ JoinPathSegments(data_root_, "hive-log4j2.properties"));
+}
} // namespace hms
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/61aed1a5/src/kudu/hms/mini_hms.h
----------------------------------------------------------------------
diff --git a/src/kudu/hms/mini_hms.h b/src/kudu/hms/mini_hms.h
index 6528e0b..b98fa4b 100644
--- a/src/kudu/hms/mini_hms.h
+++ b/src/kudu/hms/mini_hms.h
@@ -86,6 +86,9 @@ class MiniHms {
// Creates a core-site.xml for the mini HMS.
Status CreateCoreSite() const WARN_UNUSED_RESULT;
+ // Creates a log4j2 configuration properties file for the mini HMS.
+ Status CreateLogConfig() const WARN_UNUSED_RESULT;
+
// Waits for the metastore process to bind to a port.
Status WaitForHmsPorts() WARN_UNUSED_RESULT;
http://git-wip-us.apache.org/repos/asf/kudu/blob/61aed1a5/src/kudu/sentry/mini_sentry.cc
----------------------------------------------------------------------
diff --git a/src/kudu/sentry/mini_sentry.cc b/src/kudu/sentry/mini_sentry.cc
index f50aa5b..61b2810 100644
--- a/src/kudu/sentry/mini_sentry.cc
+++ b/src/kudu/sentry/mini_sentry.cc
@@ -103,6 +103,7 @@ Status MiniSentry::Start() {
// Start Sentry.
sentry_process_.reset(new Subprocess({
Substitute("$0/bin/sentry", sentry_home),
+ "--log4jConf", JoinPathSegments(tmp_dir, "log4j.properties"),
"--command", "service",
"--conffile", JoinPathSegments(tmp_dir, "sentry-site.xml"),
}));
@@ -249,11 +250,35 @@ test-admin=admin
test-user=user
kudu=admin
joe-interloper=""
-)";
+ )";
RETURN_NOT_OK(WriteStringToFile(Env::Default(), kUsers, users_ini_path));
+
+ // Configure the Sentry service to output WARN messages to the stderr
+ // console, and INFO and above to sentry.log in the data root. The console
+ // messages have a special 'SENTRY' tag included to disambiguate them from other
+ // Java component logs.
+ static const string kLogPropertiesTemplate = R"(
+log4j.appender.console = org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout = org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern = %d{HH:mm:ss.SSS} [SENTRY - %p - %t] (%F:%L) %m%n
+log4j.appender.console.Threshold = WARN
+
+log4j.appender.file = org.apache.log4j.FileAppender
+log4j.appender.file.File = $0
+log4j.appender.file.layout = org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+log4j.rootLogger = INFO, console, file
+ )";
+ string log_properties = Substitute(
+ kLogPropertiesTemplate,
+ JoinPathSegments(tmp_dir, "sentry.log"));
+ RETURN_NOT_OK(WriteStringToFile(Env::Default(),
+ log_properties,
+ JoinPathSegments(tmp_dir, "log4j.properties")));
+
return Status::OK();
}
-
} // namespace sentry
} // namespace kudu