You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/22 19:32:16 UTC
[50/51] [abbrv] hadoop git commit: YARN-5690. Integrate native
services modules into maven build. Contributed by Billie Rinaldi
YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c857ea7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c857ea7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c857ea7
Branch: refs/heads/yarn-native-services
Commit: 6c857ea778f4a04926213910438cd86077c5f5ca
Parents: 31848c7
Author: Gour Saha <go...@apache.org>
Authored: Thu Oct 27 08:50:36 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 26 ++++++++
.../assemblies/hadoop-yarn-services-api.xml | 36 +++++++++++
.../assemblies/hadoop-yarn-slider-dist.xml | 30 +++++++++
hadoop-project/pom.xml | 6 ++
hadoop-yarn-project/hadoop-yarn/bin/yarn | 30 +++++++++
.../hadoop-yarn-services-api/pom.xml | 44 +++++++++++--
.../conf/slideram-log4j.properties | 68 ++++++++++++++++++++
.../hadoop-yarn-slider-core/pom.xml | 38 +++++++++++
.../org/apache/slider/client/SliderClient.java | 29 +++++++--
.../org/apache/slider/common/SliderKeys.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 48 ++++++++++----
.../providers/agent/AgentClientProvider.java | 3 +-
.../slideram/SliderAMClientProvider.java | 15 +++--
.../TestPublishedConfigurationOutputter.java | 10 ++-
14 files changed, 343 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index c3f459c..552087c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -86,6 +86,32 @@
</includes>
</fileSet>
<fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+ <includes>
+ <include>*-sources.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory>
+ <outputDirectory>etc/hadoop</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+ <includes>
+ <include>*-sources.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/services-api</outputDirectory>
+ </fileSet>
+ <fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
new file mode 100644
index 0000000..589f724
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hadoop-yarn-services-api-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include>
+ <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include>
+ <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include>
+ <include>io.swagger:swagger-annotations</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
new file mode 100644
index 0000000..5de45a9
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
@@ -0,0 +1,30 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hadoop-yarn-slider-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a633873..409dd93 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -385,6 +385,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-slider-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 804fd1a..26d54b8 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,9 @@ function hadoop_usage
hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
hadoop_add_subcommand "rmadmin" "admin tools"
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
+ hadoop_add_subcommand "services-api" "run slider services api"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
+ hadoop_add_subcommand "slider" "run a slider app"
hadoop_add_subcommand "timelinereader" "run the timeline reader server"
hadoop_add_subcommand "timelineserver" "run the timeline server"
hadoop_add_subcommand "top" "view cluster information"
@@ -142,10 +144,38 @@ function yarncmd_case
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
+ services-api)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
+-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_HDFS_HOME}/${HDFS_DIR},\
+${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ ;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
+ slider)
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ HADOOP_CLASSNAME='org.apache.slider.Slider'
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
+-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_HDFS_HOME}/${HDFS_DIR},\
+${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ ;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 78b7855..c198f28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -159,12 +159,6 @@
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-base</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -192,6 +186,44 @@
</dependencies>
<profiles>
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-yarn-services-api</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<profile>
<id>rat</id>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
new file mode 100644
index 0000000..333859e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/slider.log
+log4j.appender.amlog.MaxFileSize=1MB
+log4j.appender.amlog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+# for debugging Slider
+#log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index d778f44..a2c67c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -332,6 +332,44 @@
<profiles>
<profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-yarn-slider-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>compile-protobuf</id>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 4b546cd..d1f88c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2162,6 +2162,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
hasServerLog4jProperties = log4jserver.isFile();
}
+ if (!hasServerLog4jProperties) {
+ // check for log4j properties in hadoop conf dir
+ String hadoopConfDir = System.getenv(ApplicationConstants.Environment
+ .HADOOP_CONF_DIR.name());
+ if (hadoopConfDir != null) {
+ File localFile = new File(hadoopConfDir, SliderKeys
+ .LOG4J_SERVER_PROP_FILENAME);
+ if (localFile.exists()) {
+ Path localFilePath = createLocalPath(localFile);
+ remoteConfPath = new Path(clusterDirectory,
+ SliderKeys.SUBMITTED_CONF_DIR);
+ Path remoteFilePath = new Path(remoteConfPath, SliderKeys
+ .LOG4J_SERVER_PROP_FILENAME);
+ copy(config, localFilePath, remoteFilePath);
+ hasServerLog4jProperties = true;
+ }
+ }
+ }
// the assumption here is that minimr cluster => this is a test run
// and the classpath can look after itself
@@ -2300,7 +2318,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// enable asserts
commandLine.enableJavaAssertions();
- // if the conf dir has a log4j-server.properties, switch to that
+ // if the conf dir has a slideram-log4j.properties, switch to that
if (hasServerLog4jProperties) {
commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
@@ -4471,14 +4489,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- if (isSet(libDir)) {
- File srcFolder = new File(libDir);
+ String[] libDirs = SliderUtils.getLibDirs();
+ if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
- // copy all jars except slider-core-<version>.jar
- tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter());
+ // copy all jars
+ tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
log.info("Uploading dependency for AM (version {}) from {} to {}",
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 1484ee3..3d25d33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -182,7 +182,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
/**
* Slider AM log4j file name : {@value}
*/
- String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties";
+ String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
/**
* Standard log4j file name : {@value}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index f773982..713cd02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -25,6 +25,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -1399,6 +1400,22 @@ public final class SliderUtils {
}
}
+ public static String getLibDir() {
+ String[] libDirs = getLibDirs();
+ if (libDirs == null || libDirs.length == 0) {
+ return null;
+ }
+ return libDirs[0];
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
/**
* Submit a JAR containing a specific class and map it
* @param providerResources provider map to build up
@@ -1962,31 +1979,34 @@ public final class SliderUtils {
/**
* Given a source folder create a tar.gz file
*
- * @param srcFolder
+ * @param libDirs
* @param tarGzipFile
*
* @throws IOException
*/
- public static void tarGzipFolder(File srcFolder, File tarGzipFile,
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folder {} to {}", srcFolder.getAbsolutePath(),
+ log.info("Tar-gzipping folders {} to {}", libDirs,
tarGzipFile.getAbsolutePath());
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
try(TarArchiveOutputStream taos =
new TarArchiveOutputStream(new GZIPOutputStream(
new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
}
- taos.flush();
- taos.closeArchiveEntry();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index fdc5be1..7ca469f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -562,8 +562,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
private void expandAgentTar(File agentPkgDir) throws IOException {
- String libDirProp =
- System.getProperty(PROPERTY_LIB_DIR);
+ String libDirProp = SliderUtils.getLibDir();
File tarFile = new File(libDirProp, AGENT_TAR);
expandTar(tarFile, agentPkgDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index e5430f2..b58d3aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -184,19 +184,20 @@ public class SliderAMClientProvider extends AbstractClientProvider
libdir,
miniClusterTestRun);
- String libDirProp =
- System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
log.info("Loading all dependencies for AM.");
// If slider.tar.gz is available in hdfs use it, else upload all jars
Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip();
if (fileSystem.isFile(dependencyLibTarGzip)) {
SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem);
} else {
- ProviderUtils.addAllDependencyJars(providerResources,
- fileSystem,
- tempPath,
- libdir,
- libDirProp);
+ for (String libDirProp : SliderUtils.getLibDirs()) {
+ ProviderUtils.addAllDependencyJars(providerResources,
+ fileSystem,
+ tempPath,
+ libdir,
+ libDirProp);
+
+ }
}
addKeytabResourceIfNecessary(fileSystem,
instanceDescription,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
index 3706182..56b84e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
@@ -82,16 +82,14 @@ public class TestPublishedConfigurationOutputter {
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
- assert output.contains(
- "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ assert output.contains("<name>key1</name><value>val1</value>");
File file = tmpDir.newFile();
configurationOutputter.save(file);
assert FileUtils.readFileToString(file, Charsets.UTF_8)
.replaceAll("( |\\r|\\n)", "")
- .contains(
- "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ .contains("<name>key1</name><value>val1</value>");
}
@Test
@@ -103,14 +101,14 @@ public class TestPublishedConfigurationOutputter {
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
- assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ assert output.contains("<name>key1</name><value>val1</value>");
File file = tmpDir.newFile();
configurationOutputter.save(file);
assert FileUtils.readFileToString(file, Charsets.UTF_8)
.replaceAll("( |\\r|\\n)", "")
- .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ .contains("<name>key1</name><value>val1</value>");
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org