You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by go...@apache.org on 2014/11/06 02:48:12 UTC
git commit: SLIDER-555 AM needs to get log aggregation friendly log4j
Repository: incubator-slider
Updated Branches:
refs/heads/develop 9f5dbb467 -> a29d825e6
SLIDER-555 AM needs to get log aggregation friendly log4j
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a29d825e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a29d825e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a29d825e
Branch: refs/heads/develop
Commit: a29d825e68b2f51dab80257ce3f4501cc2ce3e88
Parents: 9f5dbb4
Author: Gour Saha <go...@apache.org>
Authored: Wed Nov 5 17:38:19 2014 -0800
Committer: Gour Saha <go...@apache.org>
Committed: Wed Nov 5 17:38:19 2014 -0800
----------------------------------------------------------------------
.../src/conf/log4j-server.properties | 69 ++++++++++++++++++++
slider-assembly/src/conf/log4j.properties | 2 +-
.../org/apache/slider/client/SliderClient.java | 4 +-
.../org/apache/slider/common/SliderKeys.java | 10 +++
.../apache/slider/common/tools/SliderUtils.java | 29 ++++++--
5 files changed, 107 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-assembly/src/conf/log4j-server.properties
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf/log4j-server.properties b/slider-assembly/src/conf/log4j-server.properties
new file mode 100644
index 0000000..90b5b1b
--- /dev/null
+++ b/slider-assembly/src/conf/log4j-server.properties
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is a log4j config for Slider appmaster
+
+log4j.rootLogger=INFO,stderr
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stdout
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+# for debugging Slider
+#log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-assembly/src/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf/log4j.properties b/slider-assembly/src/conf/log4j.properties
index 823ffa3..fc5cc29 100644
--- a/slider-assembly/src/conf/log4j.properties
+++ b/slider-assembly/src/conf/log4j.properties
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# log4j configuration used during build and unit tests
+# log4j configuration used by Slider client (and during build and unit tests)
log4j.rootLogger=INFO,stderr
log4j.threshhold=ALL
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fcc6ef6..676d865 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1188,9 +1188,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
confDir);
}
Path localConfDirPath = SliderUtils.createLocalPath(confDir);
- log.debug("Slide configuration directory is {}; remote to be {}",
- localConfDirPath, SliderKeys.SUBMITTED_CONF_DIR);
remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
+ log.debug("Slider configuration directory is {}; remote to be {}",
+ localConfDirPath, remoteConfPath);
SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);
}
// the assumption here is that minimr cluster => this is a test run
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index f11d200..1a94d83 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -133,6 +133,16 @@ public interface SliderKeys extends SliderXmlConfKeys {
String SUBMITTED_CONF_DIR = "confdir";
/**
+ * Slider AM log4j file name
+ */
+ String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties";
+
+ /**
+ * Standard log4j file name
+ */
+ String LOG4J_PROP_FILENAME = "log4j.properties";
+
+ /**
* name of the Slider client resource
* loaded when the service is loaded.
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 82b17fd..0ff5ccb 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -415,7 +415,9 @@ public final class SliderUtils {
new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
permission);
}
- Path[] sourcePaths = new Path[srcFileCount];
+ // Slider AM now has its own log4j-server.properties file
+ List<Path> sourcePaths = new ArrayList<Path>();
+ Path serverLog4jPath = null;
for (int i = 0; i < srcFileCount; i++) {
FileStatus e = entries[i];
Path srcFile = e.getPath();
@@ -425,13 +427,32 @@ public final class SliderUtils {
log.warn(msg);
throw new IOException(msg);
}
+ if (SliderKeys.LOG4J_SERVER_PROP_FILENAME.equals(srcFile.getName())) {
+ // log4j-server.properties will be copied as log4j.properties
+ serverLog4jPath = srcFile;
+ log.debug("copying src conf file {} as {}", srcFile,
+ SliderKeys.LOG4J_PROP_FILENAME);
+ continue;
+ }
+ if (SliderKeys.LOG4J_PROP_FILENAME.equals(srcFile.getName())) {
+ // ignore log4j.properties entirely for Slider AM
+ log.debug("ignoring file {}", srcFile);
+ continue;
+ }
log.debug("copying src conf file {}", srcFile);
- sourcePaths[i] = srcFile;
+ sourcePaths.add(srcFile);
}
- log.debug("Copying {} files from {} to dest {}", srcFileCount,
+ log.debug("Copying {} files from {} to dest {}",
+ sourcePaths.size() + (serverLog4jPath != null ? 1 : 0),
srcDirPath,
destDirPath);
- FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
+ FileUtil.copy(srcFS, sourcePaths.toArray(new Path[0]), destFS, destDirPath,
+ false, true, conf);
+ // Now copy log4j-server.properties (if exists) to dest as log4j.properties
+ if (serverLog4jPath != null) {
+ FileUtil.copy(srcFS, serverLog4jPath, destFS, new Path(destDirPath,
+ SliderKeys.LOG4J_PROP_FILENAME), false, true, conf);
+ }
return srcFileCount;
}