You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by um...@apache.org on 2014/05/12 14:44:10 UTC
svn commit: r1593948 [1/2] - in
/hadoop/common/branches/HDFS-2006/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-app...
Author: umamahesh
Date: Mon May 12 12:43:59 2014
New Revision: 1593948
URL: http://svn.apache.org/r1593948
Log:
Merge from trunk to HDFS-2006 branch
Added:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/LeveldbIterator.java
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/LeveldbIterator.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/
- copied from r1593927, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueType.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm
- copied unchanged from r1593927, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/
- copied from r1593927, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/
Modified:
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/CHANGES.txt Mon May 12 12:43:59 2014
@@ -23,6 +23,9 @@ Release 2.5.0 - UNRELEASED
YARN-1757. NM Recovery. Auxiliary service support. (Jason Lowe via kasha)
+ YARN-1864. Fair Scheduler Dynamic Hierarchical User Queues (Ashwin Shankar
+ via Sandy Ryza)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -61,6 +64,18 @@ Release 2.5.0 - UNRELEASED
YARN-483. Improve documentation on log aggregation in yarn-default.xml
(Akira Ajisaka via Sandy Ryza)
+ YARN-2036. Document yarn.resourcemanager.hostname in ClusterSetup (Ray
+ Chiang via Sandy Ryza)
+
+ YARN-766. TestNodeManagerShutdown in branch-2 should use Shell to form the output path and a format
+ issue in trunk. (Contributed by Siddharth Seth)
+
+ YARN-1982. Renamed the daemon name to be TimelineServer instead of History
+ Server and deprecated the old usage. (Zhijie Shen via vinodkv)
+
+ YARN-1987. Wrapper for leveldb DBIterator to aid in handling database exceptions.
+ (Jason Lowe via kasha)
+
OPTIMIZATIONS
BUG FIXES
@@ -79,6 +94,17 @@ Release 2.5.0 - UNRELEASED
YARN-1865. ShellScriptBuilder does not check for some error conditions.
(Remus Rusanu via ivanmi)
+ YARN-738. TestClientRMTokens is failing irregularly while running all yarn
+ tests (Ming Ma via jlowe)
+
+ YARN-2018. TestClientRMService.testTokenRenewalWrongUser fails after
+ HADOOP-10562 (Ming Ma via Arpit Agarwal)
+
+ YARN-2011. Fix typo and warning in TestLeafQueue (Chen He via junping_du)
+
+ YARN-1975. Fix yarn application CLI to print the scheme of the tracking url
+ of failed/killed applications. (Junping Du via jianhe)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -89,6 +115,16 @@ Release 2.4.1 - UNRELEASED
YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen)
+ YARN-1696. Added documentation for ResourceManager fail-over. (Karthik
+ Kambatla, Masatake Iwasaki, Tsuyoshi OZAWA via vinodkv)
+
+ YARN-1701. Improved default paths of the timeline store and the generic
+ history store. (Tsuyoshi Ozawa via zjshen)
+
+ YARN-1962. Changed Timeline Service client configuration to be off by default
+ given the non-readiness of the feature yet. (Mohammad Kamrul Islam via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -161,6 +197,12 @@ Release 2.4.1 - UNRELEASED
YARN-1975. Used resources shows escaped html in CapacityScheduler and
FairScheduler page (Mit Desai via jlowe)
+ YARN-1929. Fixed a deadlock in ResourceManager that occurs when failover
+ happens right at the time of shutdown. (Karthik Kambatla via vinodkv)
+
+ YARN-1201. TestAMAuthorization fails with local hostname cannot be resolved.
+ (Wangda Tan via junping_du)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn Mon May 12 12:43:59 2014
@@ -63,7 +63,7 @@ function print_usage(){
echo "where COMMAND is one of:"
echo " resourcemanager run the ResourceManager"
echo " nodemanager run a nodemanager on each slave"
- echo " historyserver run the application history server"
+ echo " timelineserver run the timeline server"
echo " rmadmin admin tools"
echo " version print the version"
echo " jar <jar> run a jar file"
@@ -153,7 +153,6 @@ if [ -d "$HADOOP_YARN_HOME/yarn-server/y
fi
if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes
- CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes
fi
if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes
@@ -217,12 +216,21 @@ elif [ "$COMMAND" = "resourcemanager" ]
JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
fi
elif [ "$COMMAND" = "historyserver" ] ; then
+ echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
+ echo "Instead use the timelineserver command for it." 1>&2
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS"
if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m"
- fi
+ fi
+elif [ "$COMMAND" = "timelineserver" ] ; then
+ CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/timelineserver-config/log4j.properties
+ CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+ YARN_OPTS="$YARN_OPTS $YARN_TIMELINESERVER_OPTS"
+ if [ "$YARN_TIMELINESERVER_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_TIMELINESERVER_HEAPSIZE""m"
+ fi
elif [ "$COMMAND" = "nodemanager" ] ; then
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd Mon May 12 12:43:59 2014
@@ -120,11 +120,12 @@ if "%1" == "--config" (
if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes
- set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-applicationhistoryservice\target\classes
)
+
if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-applicationhistoryservice\target\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-applicationhistoryservice\target\classes
)
+
if exist %HADOOP_YARN_HOME%\build\test\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes
)
@@ -142,7 +143,8 @@ if "%1" == "--config" (
)
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
- application applicationattempt container node logs daemonlog historyserver
+ application applicationattempt container node logs daemonlog historyserver ^
+ timelineserver
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
)
@@ -204,6 +206,8 @@ goto :eof
goto :eof
:historyserver
+ @echo DEPRECATED: Use of this command to start the timeline server is deprecated. 1>&2
+ @echo Instead use the timelineserver command for it. 1>&2
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\ahs-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer
set YARN_OPTS=%YARN_OPTS% %HADOOP_HISTORYSERVER_OPTS%
@@ -212,6 +216,15 @@ goto :eof
)
goto :eof
+:timelineserver
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer
+ set YARN_OPTS=%YARN_OPTS% %HADOOP_TIMELINESERVER_OPTS%
+ if defined YARN_TIMELINESERVER_HEAPSIZE (
+ set JAVA_HEAP_MAX=-Xmx%YARN_TIMELINESERVER_HEAPSIZE%m
+ )
+ goto :eof
+
:nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
@@ -277,7 +290,7 @@ goto :eof
@echo where COMMAND is one of:
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
- @echo historyserver run the application history server
+ @echo timelineserver run the timeline server
@echo rmadmin admin tools
@echo version print the version
@echo jar ^<jar^> run a jar file
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh Mon May 12 12:43:59 2014
@@ -54,14 +54,14 @@ fi
# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
#export YARN_RESOURCEMANAGER_HEAPSIZE=1000
-# Specify the max Heapsize for the HistoryManager using a numerical value
+# Specify the max Heapsize for the timeline server using a numerical value
# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
# the value to 1000.
# This value will be overridden by an Xmx setting specified in either YARN_OPTS
-# and/or YARN_HISTORYSERVER_OPTS.
+# and/or YARN_TIMELINESERVER_OPTS.
# If not specified, the default value will be picked from either YARN_HEAPMAX
# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-#export YARN_HISTORYSERVER_HEAPSIZE=1000
+#export YARN_TIMELINESERVER_HEAPSIZE=1000
# Specify the JVM options to be used when starting the ResourceManager.
# These options will be appended to the options specified as YARN_OPTS
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml Mon May 12 12:43:59 2014
@@ -51,24 +51,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Mon May 12 12:43:59 2014
@@ -1105,7 +1105,7 @@ public class YarnConfiguration extends C
/** The setting that controls whether timeline service is enabled or not. */
public static final String TIMELINE_SERVICE_ENABLED =
TIMELINE_SERVICE_PREFIX + "enabled";
- public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = true;
+ public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = false;
/** host:port address for timeline service RPC APIs. */
public static final String TIMELINE_SERVICE_ADDRESS =
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml Mon May 12 12:43:59 2014
@@ -38,24 +38,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml Mon May 12 12:43:59 2014
@@ -50,24 +50,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Mon May 12 12:43:59 2014
@@ -35,24 +35,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java Mon May 12 12:43:59 2014
@@ -49,7 +49,9 @@ public class TestTimelineClient {
@Before
public void setup() {
- client = createTimelineClient(new YarnConfiguration());
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ client = createTimelineClient(conf);
}
@After
@@ -127,6 +129,25 @@ public class TestTimelineClient {
}
}
+ @Test
+ public void testPostEntitiesTimelineServiceDefaultNotEnabled()
+ throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ // Unset the timeline service's enabled properties.
+ // Make sure default value is pickup up
+ conf.unset(YarnConfiguration.TIMELINE_SERVICE_ENABLED);
+ TimelineClientImpl client = createTimelineClient(conf);
+ mockClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
+ false, false);
+ try {
+ TimelinePutResponse response = client.putEntities(generateEntity());
+ Assert.assertEquals(0, response.getErrors().size());
+ } catch (YarnException e) {
+ Assert
+ .fail("putEntities should already return before throwing the exception");
+ }
+ }
+
private static ClientResponse mockClientResponse(TimelineClientImpl client,
ClientResponse.Status status, boolean hasError, boolean hasRuntimeError) {
ClientResponse response = mock(ClientResponse.class);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Mon May 12 12:43:59 2014
@@ -38,24 +38,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Mon May 12 12:43:59 2014
@@ -1112,7 +1112,7 @@
If enabled, clients will put entities and events to the timeline server.
</description>
<name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -1161,7 +1161,7 @@
<property>
<description>Store file name for leveldb timeline store.</description>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>${yarn.log.dir}/timeline</value>
+ <value>${hadoop.tmp.dir}/yarn/timeline</value>
</property>
<property>
@@ -1210,7 +1210,7 @@
org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
as the value for yarn.timeline-service.generic-application-history.store-class</description>
<name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name>
- <value>${hadoop.log.dir}/yarn/system/history</value>
+ <value>${hadoop.tmp.dir}/yarn/timeline/generic-history</value>
</property>
<property>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml Mon May 12 12:43:59 2014
@@ -45,24 +45,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml Mon May 12 12:43:59 2014
@@ -38,24 +38,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
@@ -103,6 +85,10 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Mon May 12 12:43:59 2014
@@ -40,24 +40,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Mon May 12 12:43:59 2014
@@ -157,7 +157,7 @@ public class TestNodeManagerShutdown {
public static void startContainer(NodeManager nm, ContainerId cId,
FileContext localFS, File scriptFileDir, File processStartFile)
- throws IOException, YarnException {
+ throws IOException, YarnException {
File scriptFile =
createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Mon May 12 12:43:59 2014
@@ -42,24 +42,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Mon May 12 12:43:59 2014
@@ -101,7 +101,7 @@ public class AdminService extends Compos
}
@Override
- public synchronized void serviceInit(Configuration conf) throws Exception {
+ public void serviceInit(Configuration conf) throws Exception {
if (rmContext.isHAEnabled()) {
autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf);
if (autoFailoverEnabled) {
@@ -123,13 +123,13 @@ public class AdminService extends Compos
}
@Override
- protected synchronized void serviceStart() throws Exception {
+ protected void serviceStart() throws Exception {
startServer();
super.serviceStart();
}
@Override
- protected synchronized void serviceStop() throws Exception {
+ protected void serviceStop() throws Exception {
stopServer();
super.serviceStop();
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java Mon May 12 12:43:59 2014
@@ -61,7 +61,7 @@ public class EmbeddedElectorService exte
}
@Override
- protected synchronized void serviceInit(Configuration conf)
+ protected void serviceInit(Configuration conf)
throws Exception {
conf = conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf);
@@ -102,20 +102,20 @@ public class EmbeddedElectorService exte
}
@Override
- protected synchronized void serviceStart() throws Exception {
+ protected void serviceStart() throws Exception {
elector.joinElection(localActiveNodeInfo);
super.serviceStart();
}
@Override
- protected synchronized void serviceStop() throws Exception {
+ protected void serviceStop() throws Exception {
elector.quitElection(false);
elector.terminateConnection();
super.serviceStop();
}
@Override
- public synchronized void becomeActive() throws ServiceFailedException {
+ public void becomeActive() throws ServiceFailedException {
try {
rmContext.getRMAdminService().transitionToActive(req);
} catch (Exception e) {
@@ -124,7 +124,7 @@ public class EmbeddedElectorService exte
}
@Override
- public synchronized void becomeStandby() {
+ public void becomeStandby() {
try {
rmContext.getRMAdminService().transitionToStandby(req);
} catch (Exception e) {
@@ -143,13 +143,13 @@ public class EmbeddedElectorService exte
@SuppressWarnings(value = "unchecked")
@Override
- public synchronized void notifyFatalError(String errorMessage) {
+ public void notifyFatalError(String errorMessage) {
rmContext.getDispatcher().getEventHandler().handle(
new RMFatalEvent(RMFatalEventType.EMBEDDED_ELECTOR_FAILED, errorMessage));
}
@Override
- public synchronized void fenceOldActive(byte[] oldActiveData) {
+ public void fenceOldActive(byte[] oldActiveData) {
if (LOG.isDebugEnabled()) {
LOG.debug("Request to fence old active being ignored, " +
"as embedded leader election doesn't support fencing");
@@ -166,7 +166,7 @@ public class EmbeddedElectorService exte
.toByteArray();
}
- private synchronized boolean isParentZnodeSafe(String clusterId)
+ private boolean isParentZnodeSafe(String clusterId)
throws InterruptedException, IOException, KeeperException {
byte[] data;
try {
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Mon May 12 12:43:59 2014
@@ -526,7 +526,7 @@ public class RMAppAttemptImpl implements
private void setTrackingUrlToRMAppPage() {
originalTrackingUrl = pjoin(
- WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf),
+ WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
"cluster", "app", getAppAttemptId().getApplicationId());
proxiedTrackingUrl = originalTrackingUrl;
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java Mon May 12 12:43:59 2014
@@ -77,19 +77,22 @@ public class AllocationConfiguration {
@VisibleForTesting
QueuePlacementPolicy placementPolicy;
+ //Configured queues in the alloc xml
@VisibleForTesting
- Set<String> queueNames;
+ Map<FSQueueType, Set<String>> configuredQueues;
- public AllocationConfiguration(Map<String, Resource> minQueueResources,
- Map<String, Resource> maxQueueResources,
+ public AllocationConfiguration(Map<String, Resource> minQueueResources,
+ Map<String, Resource> maxQueueResources,
Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps,
Map<String, ResourceWeights> queueWeights, int userMaxAppsDefault,
- int queueMaxAppsDefault, Map<String, SchedulingPolicy> schedulingPolicies,
+ int queueMaxAppsDefault,
+ Map<String, SchedulingPolicy> schedulingPolicies,
SchedulingPolicy defaultSchedulingPolicy,
- Map<String, Long> minSharePreemptionTimeouts,
+ Map<String, Long> minSharePreemptionTimeouts,
Map<String, Map<QueueACL, AccessControlList>> queueAcls,
long fairSharePreemptionTimeout, long defaultMinSharePreemptionTimeout,
- QueuePlacementPolicy placementPolicy, Set<String> queueNames) {
+ QueuePlacementPolicy placementPolicy,
+ Map<FSQueueType, Set<String>> configuredQueues) {
this.minQueueResources = minQueueResources;
this.maxQueueResources = maxQueueResources;
this.queueMaxApps = queueMaxApps;
@@ -104,7 +107,7 @@ public class AllocationConfiguration {
this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout;
this.placementPolicy = placementPolicy;
- this.queueNames = queueNames;
+ this.configuredQueues = configuredQueues;
}
public AllocationConfiguration(Configuration conf) {
@@ -121,9 +124,12 @@ public class AllocationConfiguration {
fairSharePreemptionTimeout = Long.MAX_VALUE;
schedulingPolicies = new HashMap<String, SchedulingPolicy>();
defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
+ configuredQueues = new HashMap<FSQueueType, Set<String>>();
+ for (FSQueueType queueType : FSQueueType.values()) {
+ configuredQueues.put(queueType, new HashSet<String>());
+ }
placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
- new HashSet<String>());
- queueNames = new HashSet<String>();
+ configuredQueues);
}
/**
@@ -221,8 +227,8 @@ public class AllocationConfiguration {
return defaultSchedulingPolicy;
}
- public Set<String> getQueueNames() {
- return queueNames;
+ public Map<FSQueueType, Set<String>> getConfiguredQueues() {
+ return configuredQueues;
}
public QueuePlacementPolicy getPlacementPolicy() {
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java Mon May 12 12:43:59 2014
@@ -214,8 +214,15 @@ public class AllocationFileLoaderService
QueuePlacementPolicy newPlacementPolicy = null;
// Remember all queue names so we can display them on web UI, etc.
- Set<String> queueNamesInAllocFile = new HashSet<String>();
-
+ // configuredQueues is segregated based on whether it is a leaf queue
+ // or a parent queue. This information is used for creating queues
+ // and also for making queue placement decisions(QueuePlacementRule.java).
+ Map<FSQueueType, Set<String>> configuredQueues =
+ new HashMap<FSQueueType, Set<String>>();
+ for (FSQueueType queueType : FSQueueType.values()) {
+ configuredQueues.put(queueType, new HashSet<String>());
+ }
+
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory =
DocumentBuilderFactory.newInstance();
@@ -289,26 +296,27 @@ public class AllocationFileLoaderService
}
parent = null;
}
- loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps,
- userMaxApps, queueWeights, queuePolicies, minSharePreemptionTimeouts,
- queueAcls, queueNamesInAllocFile);
+ loadQueue(parent, element, minQueueResources, maxQueueResources,
+ queueMaxApps, userMaxApps, queueWeights, queuePolicies,
+ minSharePreemptionTimeouts, queueAcls,
+ configuredQueues);
}
// Load placement policy and pass it configured queues
Configuration conf = getConfig();
if (placementPolicyElement != null) {
newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement,
- queueNamesInAllocFile, conf);
+ configuredQueues, conf);
} else {
newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
- queueNamesInAllocFile);
+ configuredQueues);
}
AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources,
queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault,
queueMaxAppsDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts,
queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout,
- newPlacementPolicy, queueNamesInAllocFile);
+ newPlacementPolicy, configuredQueues);
lastSuccessfulReload = clock.getTime();
lastReloadAttemptFailed = false;
@@ -324,7 +332,8 @@ public class AllocationFileLoaderService
Map<String, Integer> userMaxApps, Map<String, ResourceWeights> queueWeights,
Map<String, SchedulingPolicy> queuePolicies,
Map<String, Long> minSharePreemptionTimeouts,
- Map<String, Map<QueueACL, AccessControlList>> queueAcls, Set<String> queueNamesInAllocFile)
+ Map<String, Map<QueueACL, AccessControlList>> queueAcls,
+ Map<FSQueueType, Set<String>> configuredQueues)
throws AllocationConfigurationException {
String queueName = element.getAttribute("name");
if (parentName != null) {
@@ -375,13 +384,19 @@ public class AllocationFileLoaderService
"pool".equals(field.getTagName())) {
loadQueue(queueName, field, minQueueResources, maxQueueResources,
queueMaxApps, userMaxApps, queueWeights, queuePolicies,
- minSharePreemptionTimeouts,
- queueAcls, queueNamesInAllocFile);
+ minSharePreemptionTimeouts, queueAcls, configuredQueues);
+ configuredQueues.get(FSQueueType.PARENT).add(queueName);
isLeaf = false;
}
}
if (isLeaf) {
- queueNamesInAllocFile.add(queueName);
+ // if a leaf in the alloc file is marked as type='parent'
+ // then store it under 'parent'
+ if ("parent".equals(element.getAttribute("type"))) {
+ configuredQueues.get(FSQueueType.PARENT).add(queueName);
+ } else {
+ configuredQueues.get(FSQueueType.LEAF).add(queueName);
+ }
}
queueAcls.put(queueName, acls);
if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName)
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Mon May 12 12:43:59 2014
@@ -74,7 +74,7 @@ public class QueueManager {
}
/**
- * Get a queue by name, creating it if the create param is true and is necessary.
+ * Get a leaf queue by name, creating it if the create param is true and is necessary.
* If the queue is not or can not be a leaf queue, i.e. it already exists as a
* parent queue, or one of the parents in its name is already a leaf queue,
* null is returned.
@@ -85,31 +85,53 @@ public class QueueManager {
* could be referred to as just "parent1.queue2".
*/
public FSLeafQueue getLeafQueue(String name, boolean create) {
+ FSQueue queue = getQueue(name, create, FSQueueType.LEAF);
+ if (queue instanceof FSParentQueue) {
+ return null;
+ }
+ return (FSLeafQueue) queue;
+ }
+
+ /**
+ * Get a parent queue by name, creating it if the create param is true and is necessary.
+ * If the queue is not or can not be a parent queue, i.e. it already exists as a
+ * leaf queue, or one of the parents in its name is already a leaf queue,
+ * null is returned.
+ *
+ * The root part of the name is optional, so a queue underneath the root
+ * named "queue1" could be referred to as just "queue1", and a queue named
+ * "queue2" underneath a parent named "parent1" that is underneath the root
+ * could be referred to as just "parent1.queue2".
+ */
+ public FSParentQueue getParentQueue(String name, boolean create) {
+ FSQueue queue = getQueue(name, create, FSQueueType.PARENT);
+ if (queue instanceof FSLeafQueue) {
+ return null;
+ }
+ return (FSParentQueue) queue;
+ }
+
+ private FSQueue getQueue(String name, boolean create, FSQueueType queueType) {
name = ensureRootPrefix(name);
synchronized (queues) {
FSQueue queue = queues.get(name);
if (queue == null && create) {
- FSLeafQueue leafQueue = createLeafQueue(name);
- if (leafQueue == null) {
- return null;
- }
- queue = leafQueue;
- } else if (queue instanceof FSParentQueue) {
- return null;
+ // if the queue doesn't exist,create it and return
+ queue = createQueue(name, queueType);
}
- return (FSLeafQueue)queue;
+ return queue;
}
}
/**
- * Creates a leaf queue and places it in the tree. Creates any
- * parents that don't already exist.
+ * Creates a leaf or parent queue based on what is specified in 'queueType'
+ * and places it in the tree. Creates any parents that don't already exist.
*
* @return
* the created queue, if successful. null if not allowed (one of the parent
* queues in the queue name is already a leaf queue)
*/
- private FSLeafQueue createLeafQueue(String name) {
+ private FSQueue createQueue(String name, FSQueueType queueType) {
List<String> newQueueNames = new ArrayList<String>();
newQueueNames.add(name);
int sepIndex = name.length();
@@ -143,8 +165,7 @@ public class QueueManager {
FSLeafQueue leafQueue = null;
for (int i = newQueueNames.size()-1; i >= 0; i--) {
String queueName = newQueueNames.get(i);
- if (i == 0) {
- // First name added was the leaf queue
+ if (i == 0 && queueType != FSQueueType.PARENT) {
leafQueue = new FSLeafQueue(name, scheduler, parent);
try {
leafQueue.setPolicy(queueConf.getDefaultSchedulingPolicy());
@@ -155,6 +176,7 @@ public class QueueManager {
parent.addChildQueue(leafQueue);
queues.put(leafQueue.getName(), leafQueue);
leafQueues.add(leafQueue);
+ return leafQueue;
} else {
FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent);
try {
@@ -169,53 +191,64 @@ public class QueueManager {
}
}
- return leafQueue;
+ return parent;
}
/**
- * Make way for the given leaf queue if possible, by removing incompatible
+ * Make way for the given queue if possible, by removing incompatible
* queues with no apps in them. Incompatibility could be due to
- * (1) leafToCreate being currently being a parent, or (2) an existing leaf queue in
- * the ancestry of leafToCreate.
+ * (1) queueToCreate being currently a parent but needs to change to leaf
+ * (2) queueToCreate being currently a leaf but needs to change to parent
+ * (3) an existing leaf queue in the ancestry of queueToCreate.
*
* We will never remove the root queue or the default queue in this way.
*
- * @return true if we can create leafToCreate or it already exists.
+ * @return true if we can create queueToCreate or it already exists.
*/
- private boolean removeEmptyIncompatibleQueues(String leafToCreate) {
- leafToCreate = ensureRootPrefix(leafToCreate);
+ private boolean removeEmptyIncompatibleQueues(String queueToCreate,
+ FSQueueType queueType) {
+ queueToCreate = ensureRootPrefix(queueToCreate);
- // Ensure leafToCreate is not root and doesn't have the default queue in its
+ // Ensure queueToCreate is not root and doesn't have the default queue in its
// ancestry.
- if (leafToCreate.equals(ROOT_QUEUE) ||
- leafToCreate.startsWith(
+ if (queueToCreate.equals(ROOT_QUEUE) ||
+ queueToCreate.startsWith(
ROOT_QUEUE + "." + YarnConfiguration.DEFAULT_QUEUE_NAME + ".")) {
return false;
}
- FSQueue queue = queues.get(leafToCreate);
+ FSQueue queue = queues.get(queueToCreate);
// Queue exists already.
if (queue != null) {
if (queue instanceof FSLeafQueue) {
- // If it's an already existing leaf, we're ok.
- return true;
+ if (queueType == FSQueueType.LEAF) {
+ // if queue is already a leaf then return true
+ return true;
+ }
+ // remove incompatibility since queue is a leaf currently
+ // needs to change to a parent.
+ return removeQueueIfEmpty(queue);
} else {
- // If it's an existing parent queue, remove it if it's empty.
+ if (queueType == FSQueueType.PARENT) {
+ return true;
+ }
+ // If it's an existing parent queue and needs to change to leaf,
+ // remove it if it's empty.
return removeQueueIfEmpty(queue);
}
}
// Queue doesn't exist already. Check if the new queue would be created
// under an existing leaf queue. If so, try removing that leaf queue.
- int sepIndex = leafToCreate.length();
- sepIndex = leafToCreate.lastIndexOf('.', sepIndex-1);
+ int sepIndex = queueToCreate.length();
+ sepIndex = queueToCreate.lastIndexOf('.', sepIndex-1);
while (sepIndex != -1) {
- String prefixString = leafToCreate.substring(0, sepIndex);
+ String prefixString = queueToCreate.substring(0, sepIndex);
FSQueue prefixQueue = queues.get(prefixString);
if (prefixQueue != null && prefixQueue instanceof FSLeafQueue) {
return removeQueueIfEmpty(prefixQueue);
}
- sepIndex = leafToCreate.lastIndexOf('.', sepIndex-1);
+ sepIndex = queueToCreate.lastIndexOf('.', sepIndex-1);
}
return true;
}
@@ -312,12 +345,21 @@ public class QueueManager {
}
public void updateAllocationConfiguration(AllocationConfiguration queueConf) {
- // Make sure all queues exist
- for (String name : queueConf.getQueueNames()) {
- if (removeEmptyIncompatibleQueues(name)) {
+ // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist
+ for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) {
+ if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) {
getLeafQueue(name, true);
}
}
+
+ // At this point all leaves and 'parents with at least one child' would have been created.
+ // Now create parents with no configured leaf.
+ for (String name : queueConf.getConfiguredQueues().get(
+ FSQueueType.PARENT)) {
+ if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
+ getParentQueue(name, true);
+ }
+ }
for (FSQueue queue : queues.values()) {
// Update queue metrics
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java Mon May 12 12:43:59 2014
@@ -42,17 +42,19 @@ public class QueuePlacementPolicy {
map.put("secondaryGroupExistingQueue",
QueuePlacementRule.SecondaryGroupExistingQueue.class);
map.put("specified", QueuePlacementRule.Specified.class);
+ map.put("nestedUserQueue",
+ QueuePlacementRule.NestedUserQueue.class);
map.put("default", QueuePlacementRule.Default.class);
map.put("reject", QueuePlacementRule.Reject.class);
ruleClasses = Collections.unmodifiableMap(map);
}
private final List<QueuePlacementRule> rules;
- private final Set<String> configuredQueues;
+ private final Map<FSQueueType, Set<String>> configuredQueues;
private final Groups groups;
public QueuePlacementPolicy(List<QueuePlacementRule> rules,
- Set<String> configuredQueues, Configuration conf)
+ Map<FSQueueType, Set<String>> configuredQueues, Configuration conf)
throws AllocationConfigurationException {
for (int i = 0; i < rules.size()-1; i++) {
if (rules.get(i).isTerminal()) {
@@ -72,28 +74,15 @@ public class QueuePlacementPolicy {
/**
* Builds a QueuePlacementPolicy from an xml element.
*/
- public static QueuePlacementPolicy fromXml(Element el, Set<String> configuredQueues,
- Configuration conf) throws AllocationConfigurationException {
+ public static QueuePlacementPolicy fromXml(Element el,
+ Map<FSQueueType, Set<String>> configuredQueues, Configuration conf)
+ throws AllocationConfigurationException {
List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
NodeList elements = el.getChildNodes();
for (int i = 0; i < elements.getLength(); i++) {
Node node = elements.item(i);
if (node instanceof Element) {
- Element element = (Element)node;
-
- String ruleName = element.getAttribute("name");
- if ("".equals(ruleName)) {
- throw new AllocationConfigurationException("No name provided for a " +
- "rule element");
- }
-
- Class<? extends QueuePlacementRule> clazz = ruleClasses.get(ruleName);
- if (clazz == null) {
- throw new AllocationConfigurationException("No rule class found for "
- + ruleName);
- }
- QueuePlacementRule rule = ReflectionUtils.newInstance(clazz, null);
- rule.initializeFromXml(element);
+ QueuePlacementRule rule = createAndInitializeRule(node);
rules.add(rule);
}
}
@@ -101,11 +90,37 @@ public class QueuePlacementPolicy {
}
/**
+ * Create and initialize a rule given a xml node
+ * @param node
+ * @return QueuePlacementPolicy
+ * @throws AllocationConfigurationException
+ */
+ public static QueuePlacementRule createAndInitializeRule(Node node)
+ throws AllocationConfigurationException {
+ Element element = (Element) node;
+
+ String ruleName = element.getAttribute("name");
+ if ("".equals(ruleName)) {
+ throw new AllocationConfigurationException("No name provided for a "
+ + "rule element");
+ }
+
+ Class<? extends QueuePlacementRule> clazz = ruleClasses.get(ruleName);
+ if (clazz == null) {
+ throw new AllocationConfigurationException("No rule class found for "
+ + ruleName);
+ }
+ QueuePlacementRule rule = ReflectionUtils.newInstance(clazz, null);
+ rule.initializeFromXml(element);
+ return rule;
+ }
+
+ /**
* Build a simple queue placement policy from the allow-undeclared-pools and
* user-as-default-queue configuration options.
*/
public static QueuePlacementPolicy fromConfiguration(Configuration conf,
- Set<String> configuredQueues) {
+ Map<FSQueueType, Set<String>> configuredQueues) {
boolean create = conf.getBoolean(
FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java Mon May 12 12:43:59 2014
@@ -18,16 +18,19 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.IOException;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import com.google.common.annotations.VisibleForTesting;
public abstract class QueuePlacementRule {
protected boolean create;
@@ -58,16 +61,20 @@ public abstract class QueuePlacementRule
* continue to the next rule, and null indicates that the app should be rejected.
*/
public String assignAppToQueue(String requestedQueue, String user,
- Groups groups, Collection<String> configuredQueues) throws IOException {
- String queue = getQueueForApp(requestedQueue, user, groups, configuredQueues);
- if (create || configuredQueues.contains(queue)) {
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues)
+ throws IOException {
+ String queue = getQueueForApp(requestedQueue, user, groups,
+ configuredQueues);
+ if (create || configuredQueues.get(FSQueueType.LEAF).contains(queue)
+ || configuredQueues.get(FSQueueType.PARENT).contains(queue)) {
return queue;
} else {
return "";
}
}
- public void initializeFromXml(Element el) {
+ public void initializeFromXml(Element el)
+ throws AllocationConfigurationException {
boolean create = true;
NamedNodeMap attributes = el.getAttributes();
Map<String, String> args = new HashMap<String, String>();
@@ -104,15 +111,16 @@ public abstract class QueuePlacementRule
* continue to the next rule.
*/
protected abstract String getQueueForApp(String requestedQueue, String user,
- Groups groups, Collection<String> configuredQueues) throws IOException;
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues)
+ throws IOException;
/**
* Places apps in queues by username of the submitter
*/
public static class User extends QueuePlacementRule {
@Override
- protected String getQueueForApp(String requestedQueue,
- String user, Groups groups, Collection<String> configuredQueues) {
+ protected String getQueueForApp(String requestedQueue, String user,
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
return "root." + user;
}
@@ -127,9 +135,9 @@ public abstract class QueuePlacementRule
*/
public static class PrimaryGroup extends QueuePlacementRule {
@Override
- protected String getQueueForApp(String requestedQueue,
- String user, Groups groups,
- Collection<String> configuredQueues) throws IOException {
+ protected String getQueueForApp(String requestedQueue, String user,
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues)
+ throws IOException {
return "root." + groups.getGroups(user).get(0);
}
@@ -147,12 +155,15 @@ public abstract class QueuePlacementRule
*/
public static class SecondaryGroupExistingQueue extends QueuePlacementRule {
@Override
- protected String getQueueForApp(String requestedQueue,
- String user, Groups groups,
- Collection<String> configuredQueues) throws IOException {
+ protected String getQueueForApp(String requestedQueue, String user,
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues)
+ throws IOException {
List<String> groupNames = groups.getGroups(user);
for (int i = 1; i < groupNames.size(); i++) {
- if (configuredQueues.contains("root." + groupNames.get(i))) {
+ String group = groupNames.get(i);
+ if (configuredQueues.get(FSQueueType.LEAF).contains("root." + group)
+ || configuredQueues.get(FSQueueType.PARENT).contains(
+ "root." + group)) {
return "root." + groupNames.get(i);
}
}
@@ -167,12 +178,83 @@ public abstract class QueuePlacementRule
}
/**
+ * Places apps in queues with name of the submitter under the queue
+ * returned by the nested rule.
+ */
+ public static class NestedUserQueue extends QueuePlacementRule {
+ @VisibleForTesting
+ QueuePlacementRule nestedRule;
+
+ /**
+ * Parse xml and instantiate the nested rule
+ */
+ @Override
+ public void initializeFromXml(Element el)
+ throws AllocationConfigurationException {
+ NodeList elements = el.getChildNodes();
+
+ for (int i = 0; i < elements.getLength(); i++) {
+ Node node = elements.item(i);
+ if (node instanceof Element) {
+ Element element = (Element) node;
+ if ("rule".equals(element.getTagName())) {
+ QueuePlacementRule rule = QueuePlacementPolicy
+ .createAndInitializeRule(node);
+ if (rule == null) {
+ throw new AllocationConfigurationException(
+ "Unable to create nested rule in nestedUserQueue rule");
+ }
+ this.nestedRule = rule;
+ break;
+ } else {
+ continue;
+ }
+ }
+ }
+
+ if (this.nestedRule == null) {
+ throw new AllocationConfigurationException(
+ "No nested rule specified in <nestedUserQueue> rule");
+ }
+ super.initializeFromXml(el);
+ }
+
+ @Override
+ protected String getQueueForApp(String requestedQueue, String user,
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues)
+ throws IOException {
+ // Apply the nested rule
+ String queueName = nestedRule.assignAppToQueue(requestedQueue, user,
+ groups, configuredQueues);
+
+ if (queueName != null && queueName != "") {
+ if (!queueName.startsWith("root.")) {
+ queueName = "root." + queueName;
+ }
+
+ // Verify if the queue returned by the nested rule is an configured leaf queue,
+ // if yes then skip to next rule in the queue placement policy
+ if (configuredQueues.get(FSQueueType.LEAF).contains(queueName)) {
+ return "";
+ }
+ return queueName + "." + user;
+ }
+ return queueName;
+ }
+
+ @Override
+ public boolean isTerminal() {
+ return false;
+ }
+ }
+
+ /**
* Places apps in queues by requested queue of the submitter
*/
public static class Specified extends QueuePlacementRule {
@Override
- protected String getQueueForApp(String requestedQueue,
- String user, Groups groups, Collection<String> configuredQueues) {
+ protected String getQueueForApp(String requestedQueue, String user,
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) {
return "";
} else {
@@ -195,7 +277,7 @@ public abstract class QueuePlacementRule
public static class Default extends QueuePlacementRule {
@Override
protected String getQueueForApp(String requestedQueue, String user,
- Groups groups, Collection<String> configuredQueues) {
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
}
@@ -211,13 +293,13 @@ public abstract class QueuePlacementRule
public static class Reject extends QueuePlacementRule {
@Override
public String assignAppToQueue(String requestedQueue, String user,
- Groups groups, Collection<String> configuredQueues) {
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
return null;
}
@Override
protected String getQueueForApp(String requestedQueue, String user,
- Groups groups, Collection<String> configuredQueues) {
+ Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
throw new UnsupportedOperationException();
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Mon May 12 12:43:59 2014
@@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -272,21 +273,62 @@ public class TestAMAuthorization {
client.registerApplicationMaster(request);
Assert.fail("Should fail with authorization error");
} catch (Exception e) {
- // Because there are no tokens, the request should be rejected as the
- // server side will assume we are trying simple auth.
- String expectedMessage = "";
- if (UserGroupInformation.isSecurityEnabled()) {
- expectedMessage = "Client cannot authenticate via:[TOKEN]";
+ if (isCause(AccessControlException.class, e)) {
+ // Because there are no tokens, the request should be rejected as the
+ // server side will assume we are trying simple auth.
+ String expectedMessage = "";
+ if (UserGroupInformation.isSecurityEnabled()) {
+ expectedMessage = "Client cannot authenticate via:[TOKEN]";
+ } else {
+ expectedMessage =
+ "SIMPLE authentication is not enabled. Available:[TOKEN]";
+ }
+ Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
} else {
- expectedMessage =
- "SIMPLE authentication is not enabled. Available:[TOKEN]";
+ throw e;
}
- Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
}
// TODO: Add validation of invalid authorization when there's more data in
// the AMRMToken
}
+
+ /**
+ * Identify if an expected throwable included in an exception stack. We use
+ * this because sometimes, an exception will be wrapped to another exception
+ * before thrown. Like,
+ *
+ * <pre>
+ * {@code
+ * void methodA() throws IOException {
+ * try {
+ * // something
+ * } catch (AccessControlException e) {
+ * // do process
+ * throw new IOException(e)
+ * }
+ * }
+ * </pre>
+ *
+ * So we cannot simply catch AccessControlException by using
+ * <pre>
+ * {@code
+ * try {
+ * methodA()
+ * } catch (AccessControlException e) {
+ * // do something
+ * }
+ * </pre>
+ *
+ * This method is useful in such cases.
+ */
+ private static boolean isCause(
+ Class<? extends Throwable> expected,
+ Throwable e
+ ) {
+ return (e != null)
+ && (expected.isInstance(e) || isCause(expected, e.getCause()));
+ }
private void waitForLaunchedState(RMAppAttempt attempt)
throws InterruptedException {
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Mon May 12 12:43:59 2014
@@ -478,9 +478,8 @@ public class TestClientRMService {
checkTokenRenewal(owner, other);
return null;
} catch (YarnException ex) {
- Assert.assertTrue(ex.getMessage().contains(
- "Client " + owner.getUserName() +
- " tries to renew a token with renewer specified as " +
+ Assert.assertTrue(ex.getMessage().contains(owner.getUserName() +
+ " tries to renew a token with renewer " +
other.getUserName()));
throw ex;
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java Mon May 12 12:43:59 2014
@@ -36,6 +36,7 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
+import org.apache.hadoop.net.NetUtils;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -235,8 +236,8 @@ public class TestClientRMTokens {
@Test
public void testShortCircuitRenewCancel()
throws IOException, InterruptedException {
- InetSocketAddress addr =
- new InetSocketAddress(InetAddress.getLocalHost(), 123);
+ InetSocketAddress addr = NetUtils.createSocketAddr(
+ InetAddress.getLocalHost().getHostName(), 123, null);
checkShortCircuitRenewCancel(addr, addr, true);
}
@@ -244,17 +245,19 @@ public class TestClientRMTokens {
public void testShortCircuitRenewCancelWildcardAddress()
throws IOException, InterruptedException {
InetSocketAddress rmAddr = new InetSocketAddress(123);
+ InetSocketAddress serviceAddr = NetUtils.createSocketAddr(
+ InetAddress.getLocalHost().getHostName(), rmAddr.getPort(), null);
checkShortCircuitRenewCancel(
rmAddr,
- new InetSocketAddress(InetAddress.getLocalHost(), rmAddr.getPort()),
+ serviceAddr,
true);
}
@Test
public void testShortCircuitRenewCancelSameHostDifferentPort()
throws IOException, InterruptedException {
- InetSocketAddress rmAddr =
- new InetSocketAddress(InetAddress.getLocalHost(), 123);
+ InetSocketAddress rmAddr = NetUtils.createSocketAddr(
+ InetAddress.getLocalHost().getHostName(), 123, null);
checkShortCircuitRenewCancel(
rmAddr,
new InetSocketAddress(rmAddr.getAddress(), rmAddr.getPort()+1),
@@ -264,8 +267,8 @@ public class TestClientRMTokens {
@Test
public void testShortCircuitRenewCancelDifferentHostSamePort()
throws IOException, InterruptedException {
- InetSocketAddress rmAddr =
- new InetSocketAddress(InetAddress.getLocalHost(), 123);
+ InetSocketAddress rmAddr = NetUtils.createSocketAddr(
+ InetAddress.getLocalHost().getHostName(), 123, null);
checkShortCircuitRenewCancel(
rmAddr,
new InetSocketAddress("1.1.1.1", rmAddr.getPort()),
@@ -275,8 +278,8 @@ public class TestClientRMTokens {
@Test
public void testShortCircuitRenewCancelDifferentHostDifferentPort()
throws IOException, InterruptedException {
- InetSocketAddress rmAddr =
- new InetSocketAddress(InetAddress.getLocalHost(), 123);
+ InetSocketAddress rmAddr = NetUtils.createSocketAddr(
+ InetAddress.getLocalHost().getHostName(), 123, null);
checkShortCircuitRenewCancel(
rmAddr,
new InetSocketAddress("1.1.1.1", rmAddr.getPort()+1),
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Mon May 12 12:43:59 2014
@@ -113,7 +113,7 @@ public class TestRMAppAttemptTransitions
private static final String EMPTY_DIAGNOSTICS = "";
private static final String RM_WEBAPP_ADDR =
- WebAppUtils.getResolvedRMWebAppURLWithoutScheme(new Configuration());
+ WebAppUtils.getResolvedRMWebAppURLWithScheme(new Configuration());
private boolean isSecurityEnabled;
private RMContext rmContext;