You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by aw...@apache.org on 2020/04/04 21:34:12 UTC
[kudu] 01/02: ranger: enable log4j2 logging to files
This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit c0a9fdc7384ac24258411949303ae1881358c3f3
Author: Andrew Wong <aw...@cloudera.com>
AuthorDate: Fri Apr 3 00:38:52 2020 -0700
ranger: enable log4j2 logging to files
This patch enables Kudu to generate a log4j2 properties file and pass it
to the Ranger client. This file is created in a directory specified by
the new --ranger_logging_config_dir flag, which, by default, will direct
the properties file to --log_dir. If the properties file already exists,
the existing file is honored. This means a file will not be regenerated
if one has already been generated -- I will add a follow up patch to
allow users to overwrite the file.
This properties file will direct logs to the directory pointed to by
--log_dir, and will honor the existing --max_log_files and
--max_log_size flags.
I've found logging to stdout and adjusting log42j log-level to be
useful, so this also adds --ranger_logtostdout and --ranger_log_level
for these respectively. I opted not to use the glog --minloglevel flag
since they don't quite map to log4j2 log levels (e.g. glog doesn't have
debug-level logging).
I manually tested that an existing properties file that has been
modified will be honored. Here's what's in a Ranger-configured Kudu
master's log directory:
kudu-master.INFO -> kudu-master.awong-MBP16-21621.local.awong.log.INFO.20200403-122527.17165
kudu-master.awong-MBP16-21621.local.awong.diagnostics.20200403-122527.0.17165
kudu-master.awong-MBP16-21621.local.awong.log.INFO.20200403-122527.17165
kudu-ranger-subprocess-log4j2.properties
kudu-ranger-subprocess.awong-MBP16-21621.local.log
Change-Id: I7efa631832c219fce214304538e6ab6442062752
Reviewed-on: http://gerrit.cloudera.org:8080/15628
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Andrew Wong <aw...@cloudera.com>
---
java/kudu-subprocess/build.gradle | 2 +
src/kudu/ranger/mini_ranger.cc | 12 +-
src/kudu/ranger/mini_ranger.h | 4 +-
src/kudu/ranger/ranger_client-test.cc | 68 +++++++++++
src/kudu/ranger/ranger_client.cc | 198 +++++++++++++++++++++++---------
src/kudu/subprocess/CMakeLists.txt | 1 +
src/kudu/subprocess/subprocess_proxy.cc | 128 +++++++++++++++++++++
src/kudu/subprocess/subprocess_proxy.h | 23 +++-
src/kudu/util/env.h | 3 +
9 files changed, 374 insertions(+), 65 deletions(-)
diff --git a/java/kudu-subprocess/build.gradle b/java/kudu-subprocess/build.gradle
index eafaa83..1a9cdca 100644
--- a/java/kudu-subprocess/build.gradle
+++ b/java/kudu-subprocess/build.gradle
@@ -32,6 +32,8 @@ dependencies {
exclude group: "org.apache.kafka"
}
compile libs.slf4jApi
+ compile libs.log4j
+ compile libs.log4jSlf4jImpl
// Workaround for RANGER-2749. Remove once resolved.
compile "commons-lang:commons-lang:2.6"
diff --git a/src/kudu/ranger/mini_ranger.cc b/src/kudu/ranger/mini_ranger.cc
index a880a97..9d5c284 100644
--- a/src/kudu/ranger/mini_ranger.cc
+++ b/src/kudu/ranger/mini_ranger.cc
@@ -316,19 +316,19 @@ Status MiniRanger::PostToRanger(string url, EasyJson payload) {
return Status::OK();
}
-Status MiniRanger::CreateClientConfig(const string& client_config_path) {
- auto policy_cache = JoinPathSegments(client_config_path, "policy-cache");
- if (!env_->FileExists(client_config_path)) {
- RETURN_NOT_OK(env_->CreateDir(client_config_path));
+Status MiniRanger::CreateClientConfig(const string& client_config_dir) {
+ auto policy_cache = JoinPathSegments(client_config_dir, "policy-cache");
+ if (!env_->FileExists(client_config_dir)) {
+ RETURN_NOT_OK(env_->CreateDir(client_config_dir));
RETURN_NOT_OK(env_->CreateDir(policy_cache));
}
RETURN_NOT_OK(WriteStringToFile(env_, GetRangerCoreSiteXml(kerberos_),
- JoinPathSegments(client_config_path, "core-site.xml")));
+ JoinPathSegments(client_config_dir, "core-site.xml")));
RETURN_NOT_OK(WriteStringToFile(env_, GetRangerKuduSecurityXml(policy_cache, "kudu",
ranger_admin_url_,
policy_poll_interval_ms_),
- JoinPathSegments(client_config_path,
+ JoinPathSegments(client_config_dir,
"ranger-kudu-security.xml")));
return Status::OK();
}
diff --git a/src/kudu/ranger/mini_ranger.h b/src/kudu/ranger/mini_ranger.h
index e6ad364..7449f13 100644
--- a/src/kudu/ranger/mini_ranger.h
+++ b/src/kudu/ranger/mini_ranger.h
@@ -86,8 +86,8 @@ class MiniRanger {
// Adds a new policy to Ranger.
Status AddPolicy(AuthorizationPolicy policy) WARN_UNUSED_RESULT;
- // Creates the client configs files in client_config_path.
- Status CreateClientConfig(const std::string& client_config_path) WARN_UNUSED_RESULT;
+ // Creates the client configs files in the given directory.
+ Status CreateClientConfig(const std::string& client_config_dir) WARN_UNUSED_RESULT;
void EnableKerberos(std::string krb5_config,
std::string admin_ktpath,
diff --git a/src/kudu/ranger/ranger_client-test.cc b/src/kudu/ranger/ranger_client-test.cc
index 017423e..73d0128 100644
--- a/src/kudu/ranger/ranger_client-test.cc
+++ b/src/kudu/ranger/ranger_client-test.cc
@@ -22,27 +22,41 @@
#include <string>
#include <unordered_set>
#include <utility>
+#include <vector>
#include <boost/functional/hash/hash.hpp>
+#include <gflags/gflags_declare.h>
#include <glog/logging.h>
#include <google/protobuf/any.pb.h>
#include <gtest/gtest.h>
#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/join.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/ranger/mini_ranger.h"
#include "kudu/ranger/ranger.pb.h"
#include "kudu/subprocess/server.h"
#include "kudu/subprocess/subprocess.pb.h"
#include "kudu/util/env.h"
+#include "kudu/util/env_util.h"
#include "kudu/util/metrics.h"
+#include "kudu/util/path_util.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
+DECLARE_string(log_dir);
+DECLARE_string(ranger_config_path);
+DECLARE_string(ranger_log_config_dir);
+DECLARE_string(ranger_log_level);
+DECLARE_bool(ranger_logtostdout);
+
namespace kudu {
namespace ranger {
using boost::hash_combine;
+using kudu::env_util::ListFilesInDir;
using kudu::subprocess::SubprocessMetrics;
using kudu::subprocess::SubprocessRequestPB;
using kudu::subprocess::SubprocessResponsePB;
@@ -50,6 +64,7 @@ using kudu::subprocess::SubprocessServer;
using std::move;
using std::string;
using std::unordered_set;
+using std::vector;
using strings::Substitute;
struct AuthorizedAction {
@@ -308,5 +323,58 @@ TEST_F(RangerClientTest, TestAuthorizeActionsAllAuthorized) {
ASSERT_EQ(3, actions.size());
}
+class RangerClientTestBase : public KuduTest {
+ public:
+ RangerClientTestBase()
+ : test_dir_(GetTestDataDirectory()) {}
+
+ void SetUp() override {
+ metric_entity_ = METRIC_ENTITY_server.Instantiate(&metric_registry_, "ranger_client-test");
+ FLAGS_ranger_log_level = "debug";
+ FLAGS_ranger_logtostdout = true;
+ FLAGS_ranger_config_path = test_dir_;
+ FLAGS_ranger_log_config_dir = JoinPathSegments(test_dir_, "log_conf");
+ FLAGS_log_dir = JoinPathSegments(test_dir_, "logs");
+ ASSERT_OK(env_->CreateDir(FLAGS_log_dir));
+ ASSERT_OK(InitializeRanger());
+ }
+
+ Status InitializeRanger() {
+ ranger_.reset(new MiniRanger("127.0.0.1"));
+ RETURN_NOT_OK(ranger_->Start());
+ RETURN_NOT_OK(ranger_->CreateClientConfig(test_dir_));
+ client_.reset(new RangerClient(env_, metric_entity_));
+ return client_->Start();
+ }
+
+ protected:
+ const string test_dir_;
+ MetricRegistry metric_registry_;
+
+ scoped_refptr<MetricEntity> metric_entity_;
+ std::unique_ptr<MiniRanger> ranger_;
+ std::unique_ptr<RangerClient> client_;
+};
+
+TEST_F(RangerClientTestBase, TestLogging) {
+ {
+ // Check that a logging configuration was produced by the Ranger client.
+ vector<string> files;
+ ASSERT_OK(ListFilesInDir(env_, FLAGS_ranger_log_config_dir, &files));
+ SCOPED_TRACE(JoinStrings(files, "\n"));
+ ASSERT_STRINGS_ANY_MATCH(files, ".*log4j2.properties");
+ }
+ // Make a request. It doesn't matter whether it succeeds or not -- debug logs
+ // should include info about each request.
+ Status s = client_->AuthorizeAction("user", ActionPB::ALL, "table");
+ ASSERT_TRUE(s.IsNotAuthorized());
+
+ // Check that the Ranger client logs some things.
+ vector<string> files;
+ ASSERT_OK(ListFilesInDir(env_, FLAGS_log_dir, &files));
+ SCOPED_TRACE(JoinStrings(files, "\n"));
+ ASSERT_STRINGS_ANY_MATCH(files, "kudu-ranger-subprocess.*log");
+}
+
} // namespace ranger
} // namespace kudu
diff --git a/src/kudu/ranger/ranger_client.cc b/src/kudu/ranger/ranger_client.cc
index be40dd1..07baa40 100644
--- a/src/kudu/ranger/ranger_client.cc
+++ b/src/kudu/ranger/ranger_client.cc
@@ -17,6 +17,7 @@
#include "kudu/ranger/ranger_client.h"
+#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <ostream>
@@ -39,9 +40,11 @@
#include "kudu/util/flag_tags.h"
#include "kudu/util/flag_validators.h"
#include "kudu/util/metrics.h"
+#include "kudu/util/net/net_util.h"
#include "kudu/util/path_util.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
+#include "kudu/util/string_case.h"
#include "kudu/util/subprocess.h"
DEFINE_string(ranger_java_path, "",
@@ -72,6 +75,29 @@ DEFINE_string(ranger_receiver_fifo_dir, "",
"--ranger_config_path directory.");
TAG_FLAG(ranger_receiver_fifo_dir, advanced);
+DEFINE_string(ranger_log_config_dir, "",
+ "Directory in which to look for a kudu-ranger-subprocess-log4j2.properties "
+ "file. If empty, will use the value of --log_dir. If such a file does not "
+ "exist, a properties file will be created to honor Kudu's logging "
+ "configurations.");
+TAG_FLAG(ranger_log_config_dir, advanced);
+TAG_FLAG(ranger_log_config_dir, evolving);
+
+DEFINE_string(ranger_log_level, "info",
+ "Log level to use in the Ranger Java subprocess. Supports \"all\", \"trace\", "
+ "\"debug\", \"info\", \"warn\", \"error\", \"fatal\", and \"off\"");
+TAG_FLAG(ranger_log_level, advanced);
+TAG_FLAG(ranger_log_level, evolving);
+
+DEFINE_bool(ranger_logtostdout, false,
+ "Whether to have the Ranger subprocess log to stdout.");
+TAG_FLAG(ranger_logtostdout, advanced);
+TAG_FLAG(ranger_logtostdout, evolving);
+
+DECLARE_int32(max_log_files);
+DECLARE_int32(max_log_size);
+DECLARE_string(log_dir);
+
METRIC_DEFINE_histogram(server, ranger_subprocess_execution_time_ms,
"Ranger subprocess execution time (ms)",
kudu::MetricUnit::kMilliseconds,
@@ -138,20 +164,34 @@ namespace ranger {
using kudu::security::GetKrb5ConfigFile;
using kudu::subprocess::SubprocessMetrics;
+using kudu::subprocess::SubprocessServer;
using std::move;
using std::string;
using std::unordered_set;
using std::vector;
using strings::Substitute;
-static const char* kUnauthorizedAction = "Unauthorized action";
-static const char* kDenyNonRangerTableTemplate = "Denying action on table with invalid name $0. "
- "Use 'kudu table rename_table' to rename it to "
- "a Ranger-compatible name.";
-static const char* kMainClass = "org.apache.kudu.subprocess.ranger.RangerSubprocessMain";
+namespace {
+
+const char* kUnauthorizedAction = "Unauthorized action";
+const char* kDenyNonRangerTableTemplate = "Denying action on table with invalid name $0. "
+ "Use 'kudu table rename_table' to rename it to "
+ "a Ranger-compatible name.";
+const char* kMainClass = "org.apache.kudu.subprocess.ranger.RangerSubprocessMain";
+const char* kRangerClientLogFilename = "kudu-ranger-subprocess";
+const char* kRangerClientPropertiesFilename = "kudu-ranger-subprocess-log4j2.properties";
+
+const char* ScopeToString(RangerClient::Scope scope) {
+ switch (scope) {
+ case RangerClient::Scope::DATABASE: return "database";
+ case RangerClient::Scope::TABLE: return "table";
+ }
+ LOG(FATAL) << static_cast<uint16_t>(scope) << ": unknown scope";
+ __builtin_unreachable();
+}
// Returns the path to the JAR file containing the Ranger subprocess.
-static string GetRangerJarPath() {
+string RangerJarPath() {
if (FLAGS_ranger_jar_path.empty()) {
string exe;
CHECK_OK(Env::Default()->GetExecutablePath(&exe));
@@ -162,18 +202,19 @@ static string GetRangerJarPath() {
}
// Returns the classpath to be used for the Ranger subprocess.
-static string GetJavaClasspath() {
- return Substitute("$0:$1", GetRangerJarPath(), FLAGS_ranger_config_path);
+string JavaClasspath() {
+ DCHECK(!FLAGS_ranger_config_path.empty());
+ return Substitute("$0:$1", RangerJarPath(), FLAGS_ranger_config_path);
}
-static string ranger_fifo_base() {
+string RangerFifoBase() {
DCHECK(!FLAGS_ranger_config_path.empty());
const string& fifo_dir = FLAGS_ranger_receiver_fifo_dir.empty() ?
FLAGS_ranger_config_path : FLAGS_ranger_receiver_fifo_dir;
return JoinPathSegments(fifo_dir, "ranger_receiever_fifo");
}
-static string java_path() {
+string JavaPath() {
if (FLAGS_ranger_java_path.empty()) {
auto java_home = getenv("JAVA_HOME");
if (!java_home) {
@@ -184,20 +225,103 @@ static string java_path() {
return FLAGS_ranger_java_path;
}
+bool ValidateRangerConfiguration() {
+ if (!FLAGS_ranger_config_path.empty()) {
+ // First, check the specified Java path.
+ const string java_path = JavaPath();
+ if (!Env::Default()->FileExists(java_path)) {
+ // Otherwise, since the specified path is not absolute, check if
+ // the Java binary is on the PATH.
+ string p;
+ Status s = Subprocess::Call({ "which", java_path }, "", &p);
+ if (!s.ok()) {
+ LOG(ERROR) << Substitute("--ranger_java_path has invalid java binary path: $0",
+ java_path);
+ return false;
+ }
+ }
+ const string ranger_jar_path = RangerJarPath();
+ if (!Env::Default()->FileExists(ranger_jar_path)) {
+ LOG(ERROR) << Substitute("--ranger_jar_path has invalid JAR file path: $0",
+ ranger_jar_path);
+ return false;
+ }
+ }
+ return true;
+}
+GROUP_FLAG_VALIDATOR(ranger_config_flags, ValidateRangerConfiguration);
+
+bool ValidateLog4jLevel(const char* /*flagname*/, const string& value) {
+ static const vector<string> kLevels = {
+ "all",
+ "trace",
+ "debug",
+ "info",
+ "warn",
+ "error",
+ "fatal",
+ "off",
+ };
+ string vlower = value;
+ ToLowerCase(&vlower);
+ if (std::any_of(kLevels.begin(), kLevels.end(),
+ [&vlower] (const string& level) { return level == vlower; })) {
+ return true;
+ }
+ LOG(ERROR) << Substitute("expected one of {$0} but got $1",
+ JoinStrings(kLevels, ", "), value);
+ return false;
+}
+DEFINE_validator(ranger_log_level, &ValidateLog4jLevel);
+
+Status GetOrCreateLog4j2PropertiesFile(Env* env, string* logging_properties_file) {
+ const string log_conf_dir = FLAGS_ranger_log_config_dir.empty() ?
+ FLAGS_log_dir : FLAGS_ranger_log_config_dir;
+ // It's generally expected that --log_dir has already been created elsewhere.
+ if (!FLAGS_ranger_log_config_dir.empty() && !env->FileExists(log_conf_dir)) {
+ RETURN_NOT_OK(env->CreateDir(log_conf_dir));
+ }
+ const string log4j2_properties_file = JoinPathSegments(log_conf_dir,
+ kRangerClientPropertiesFilename);
+ string new_or_existing;
+ if (env->FileExists(log4j2_properties_file)) {
+ new_or_existing = "existing";
+ } else {
+ string exe;
+ RETURN_NOT_OK(env->GetExecutablePath(&exe));
+ const string program_name = BaseName(exe);
+ string hostname;
+ RETURN_NOT_OK(GetHostname(&hostname));
+ const string log_filename = Substitute("$0.$1", kRangerClientLogFilename, hostname);
+ RETURN_NOT_OK(WriteStringToFileSync(
+ env, subprocess::Log4j2Properties(program_name, FLAGS_log_dir, log_filename,
+ FLAGS_max_log_size, FLAGS_max_log_files,
+ FLAGS_ranger_log_level,
+ FLAGS_ranger_logtostdout),
+ log4j2_properties_file));
+ new_or_existing = "new";
+ }
+ LOG(INFO) << Substitute("Using $0 properties file: $1",
+ new_or_existing, log4j2_properties_file);
+ *logging_properties_file = log4j2_properties_file;
+ return Status::OK();
+}
+
// Builds the arguments to start the Ranger subprocess with the given receiver
// fifo path. Specifically pass the principal and keytab file that the Ranger
// subprocess will log in with if Kerberos is enabled. 'args' has the final
// arguments. Returns 'OK' if arguments successfully created, error otherwise.
-static Status BuildArgv(const string& fifo_path, vector<string>* argv) {
+Status BuildArgv(const string& fifo_path, const string& log_properties_file,
+ vector<string>* argv) {
DCHECK(argv);
DCHECK(!FLAGS_ranger_config_path.empty());
// Pass the required arguments to run the Ranger subprocess.
- vector<string> ret = { java_path() };
-
- ret.emplace_back(Substitute("-Djava.security.krb5.conf=$0", GetKrb5ConfigFile()));
- ret.emplace_back("-cp");
- ret.emplace_back(GetJavaClasspath());
- ret.emplace_back(kMainClass);
+ vector<string> ret = {
+ JavaPath(),
+ Substitute("-Djava.security.krb5.conf=$0", GetKrb5ConfigFile()),
+ Substitute("-Dlog4j2.configurationFile=$0", log_properties_file),
+ "-cp", JavaClasspath(), kMainClass,
+ };
// When Kerberos is enabled in Kudu, pass both Kudu principal and keytab file
// to the Ranger subprocess.
if (!FLAGS_keytab_file.empty()) {
@@ -215,39 +339,7 @@ static Status BuildArgv(const string& fifo_path, vector<string>* argv) {
return Status::OK();
}
-static bool ValidateRangerConfiguration() {
- if (!FLAGS_ranger_config_path.empty()) {
- // First, check the specified path.
- if (!Env::Default()->FileExists(java_path())) {
- // Otherwise, since the specified path is not absolute, check if
- // the Java binary is on the PATH.
- string p;
- Status s = Subprocess::Call({ "which", java_path() }, "", &p);
- if (!s.ok()) {
- LOG(ERROR) << Substitute("--ranger_java_path has invalid java binary path: $0",
- java_path());
- return false;
- }
- }
- string ranger_jar_path = GetRangerJarPath();
- if (!Env::Default()->FileExists(ranger_jar_path)) {
- LOG(ERROR) << Substitute("--ranger_jar_path has invalid JAR file path: $0",
- ranger_jar_path);
- return false;
- }
- }
- return true;
-}
-GROUP_FLAG_VALIDATOR(ranger_config_flags, ValidateRangerConfiguration);
-
-static const char* ScopeToString(RangerClient::Scope scope) {
- switch (scope) {
- case RangerClient::Scope::DATABASE: return "database";
- case RangerClient::Scope::TABLE: return "table";
- }
- LOG(FATAL) << static_cast<uint16_t>(scope) << ": unknown scope";
- __builtin_unreachable();
-}
+} // anonymous namespace
#define HISTINIT(member, x) member = METRIC_##x.Instantiate(entity)
RangerSubprocessMetrics::RangerSubprocessMetrics(const scoped_refptr<MetricEntity>& entity) {
@@ -270,9 +362,11 @@ RangerClient::RangerClient(Env* env, const scoped_refptr<MetricEntity>& metric_e
Status RangerClient::Start() {
VLOG(1) << "Initializing Ranger subprocess server";
+ string log_properties_file;
+ RETURN_NOT_OK(GetOrCreateLog4j2PropertiesFile(env_, &log_properties_file));
+ const string fifo_path = SubprocessServer::FifoPath(RangerFifoBase());
vector<string> argv;
- const string fifo_path = subprocess::SubprocessServer::FifoPath(ranger_fifo_base());
- RETURN_NOT_OK(BuildArgv(fifo_path, &argv));
+ RETURN_NOT_OK(BuildArgv(fifo_path, log_properties_file, &argv));
subprocess_.reset(new RangerSubprocess(env_, fifo_path, std::move(argv), metric_entity_));
return subprocess_->Start();
}
diff --git a/src/kudu/subprocess/CMakeLists.txt b/src/kudu/subprocess/CMakeLists.txt
index a2612db..65cb40c 100644
--- a/src/kudu/subprocess/CMakeLists.txt
+++ b/src/kudu/subprocess/CMakeLists.txt
@@ -49,6 +49,7 @@ add_custom_target(subprocess_jar DEPENDS ${SUBPROCESS_JAR})
add_library(kudu_subprocess
server.cc
subprocess_protocol.cc
+ subprocess_proxy.cc
)
target_link_libraries(kudu_subprocess
gutil
diff --git a/src/kudu/subprocess/subprocess_proxy.cc b/src/kudu/subprocess/subprocess_proxy.cc
new file mode 100644
index 0000000..ee8d3a9
--- /dev/null
+++ b/src/kudu/subprocess/subprocess_proxy.cc
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/subprocess/subprocess_proxy.h"
+
+#include <string>
+#include <vector>
+
+#include "kudu/gutil/strings/join.h"
+#include "kudu/gutil/strings/substitute.h"
+
+using std::string;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+namespace subprocess {
+
+namespace {
+// Helpers to generate log4j2 properties suitable for a Java-based subprocess.
+// TODO(awong): if our logging requirements get more complex, we should
+// consider a builder for these.
+
+// $0: appender instance
+// $1: appender name
+// $2: log directory
+// $3: log filename base - new logs will be called "{log directory}/{log
+// filename base}.log" and will roll to include the date in the name
+// $4: rollover policy size in MB
+// $5: number of files max
+const char* kLog4j2RollingPropertiesTemplate = R"(
+appender.$0.type = RollingFile
+appender.$0.name = $1
+appender.$0.layout.type = PatternLayout
+appender.$0.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+appender.$0.filename = $2/$3.log
+appender.$0.filePattern = $2/$3.%d{yyyyMMdd-HHmmss}.log.gz
+appender.$0.policies.type = Policies
+appender.$0.policies.size.type = SizeBasedTriggeringPolicy
+appender.$0.policies.size.size = $4 MB
+appender.$0.strategy.type = DefaultRolloverStrategy
+appender.$0.strategy.max = $5
+)";
+
+// $0: appender instance
+// $1: appender name
+const char* kLog4j2ConsoleProperties = R"(
+appender.$0.type = Console
+appender.$0.name = $1
+appender.$0.layout.type = PatternLayout
+appender.$0.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+)";
+
+// $0: name of the creator of these properties (e.g. the program name)
+// $1: comma-separated list of appender instances
+// $2: newline-separated list of appender configs
+// $3: log level (supports "all", "debug", "info", "warn", "error", "fatal")
+// $4: comma-separated list of appender refs
+// $5: newline-separated list of appender ref name specifications
+const char* kLog4j2PropertiesTemplate = R"(
+# THIS FILE WAS GENERATED BY $0.
+
+status = error
+name = PropertiesConfig
+appenders = $1
+$2
+rootLogger.level = $3
+rootLogger.appenderRefs = $4
+$5
+)";
+} // anonymous namespace
+
+string Log4j2Properties(const string& creator, const string& log_dir,
+ const string& log_filename, int rollover_size_mb,
+ int max_files, const string& log_level,
+ bool log_to_stdout) {
+ static const char* kRollingInstance = "rollingFile";
+ static const char* kRollingName = "RollingFileAppender";
+ static const char* kRollingRef = "rolling";
+ vector<string> appender_instances = { kRollingInstance };
+ vector<string> appender_names = { kRollingName };
+ vector<string> appender_configs = {
+ Substitute(kLog4j2RollingPropertiesTemplate, kRollingInstance, kRollingName,
+ log_dir, log_filename, rollover_size_mb, max_files)
+ };
+ vector<string> appender_refs = { kRollingRef };
+ if (log_to_stdout) {
+ static const char* kConsoleInstance = "console";
+ static const char* kConsoleName = "SystemOutAppender";
+ static const char* kConsoleRef = "stdout";
+ appender_instances.emplace_back(kConsoleInstance);
+ appender_names.emplace_back(kConsoleName);
+ appender_refs.emplace_back(kConsoleRef);
+ appender_configs.emplace_back(
+ strings::Substitute(kLog4j2ConsoleProperties, kConsoleInstance, kConsoleName));
+ }
+
+ DCHECK_EQ(appender_refs.size(), appender_names.size());
+ vector<string> appender_ref_name_specs(appender_refs.size());
+ for (int i = 0; i < appender_refs.size(); i++) {
+ appender_ref_name_specs.emplace_back(
+ strings::Substitute("rootLogger.appenderRef.$0.ref = $1",
+ appender_refs[i], appender_names[i]));
+ }
+ return Substitute(kLog4j2PropertiesTemplate, creator,
+ JoinStrings(appender_instances, ", "),
+ JoinStrings(appender_configs, "\n"),
+ log_level,
+ JoinStrings(appender_refs, ", "),
+ JoinStrings(appender_ref_name_specs, "\n"));
+}
+
+} // namespace subprocess
+} // namespace kudu
diff --git a/src/kudu/subprocess/subprocess_proxy.h b/src/kudu/subprocess/subprocess_proxy.h
index 76289bd..d5a5d1b 100644
--- a/src/kudu/subprocess/subprocess_proxy.h
+++ b/src/kudu/subprocess/subprocess_proxy.h
@@ -16,24 +16,37 @@
// under the License.
#pragma once
-#include <vector>
+#include <memory>
+#include <ostream>
#include <string>
+#include <utility>
+#include <vector>
#include <glog/logging.h>
+#include <google/protobuf/any.pb.h>
#include "kudu/common/wire_protocol.h"
-#include "kudu/subprocess/server.h"
-#include "kudu/subprocess/subprocess.pb.h"
#include "kudu/gutil/ref_counted.h"
#include "kudu/gutil/strings/substitute.h"
-#include "kudu/util/metrics.h"
+#include "kudu/subprocess/server.h"
+#include "kudu/subprocess/subprocess.pb.h"
#include "kudu/util/pb_util.h"
#include "kudu/util/status.h"
-#include "kudu/util/thread.h"
namespace kudu {
+
+class Env;
+class MetricEntity;
+
namespace subprocess {
+// Return a string that can serve as the contents of a log4j2 properties file,
+// with the given logging parameters.
+std::string Log4j2Properties(const std::string& creator, const std::string& log_dir,
+ const std::string& log_filename, int rollover_size_mb,
+ int max_files, const std::string& log_level,
+ bool log_to_stdout);
+
// Template that wraps a SubprocessServer, exposing only the underlying ReqPB
// and RespPB as an interface. The given MetricsPB will be initialized,
// allowing for metrics specific to each specialized SubprocessServer.
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index 7c57bb3..611de80 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -691,6 +691,9 @@ class FileLock {
// A utility routine: write "data" to the named file.
extern Status WriteStringToFile(Env* env, const Slice& data,
const std::string& fname);
+// Like above but also fsyncs the new file.
+extern Status WriteStringToFileSync(Env* env, const Slice& data,
+ const std::string& fname);
// A utility routine: read contents of named file into *data
extern Status ReadFileToString(Env* env, const std::string& fname,