You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2013/03/06 20:15:27 UTC
svn commit: r1453486 [1/2] - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/ hadoop-yarn/bin/ hadoop-yarn/conf/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-applications/ hadoop-yarn/hadoop-...
Author: suresh
Date: Wed Mar 6 19:15:18 2013
New Revision: 1453486
URL: http://svn.apache.org/r1453486
Log:
HADOOP-8952. Enhancements to support Hadoop on Windows Server and Windows Azure environments. Contributed by Ivan Mitic, Chuan Liu, Ramya Sunil, Bikas Saha, Kanna Karanam, John Gordon, Brandon Li, Chris Nauroth, David Lao, Sumadhur Reddy Bolli, Arpit Agarwal, Ahmed El Baz, Mike Liddell, Jing Zhao, Thejas Nair, Steve Maine, Ganeshan Iyer, Raja Aluri, Giridharan Kesavan, Ramya Bharathi Nimmagadda.
Added:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.branch-trunk-win.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/pom.xml
hadoop/common/trunk/hadoop-yarn-project/pom.xml
Added: hadoop/common/trunk/hadoop-yarn-project/CHANGES.branch-trunk-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.branch-trunk-win.txt?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.branch-trunk-win.txt (added)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.branch-trunk-win.txt Wed Mar 6 19:15:18 2013
@@ -0,0 +1,29 @@
+branch-trunk-win changes - unreleased
+
+ YARN-158. Yarn creating package-info.java must not depend on sh.
+ (Chris Nauroth via suresh)
+
+ YARN-176. Some YARN tests fail to find winutils. (Chris Nauroth via suresh)
+
+ YARN-207. YARN distribution build fails on Windows. (Chris Nauroth via
+ suresh)
+
+ YARN-199. Yarn cmd line scripts for windows. (Ivan Mitic via suresh)
+
+ YARN-213. YARN build script would be more readable using abspath.
+ (Chris Nauroth via suresh)
+
+ YARN-233. Added support for running containers in MS Windows to YARN. (Chris
+ Nauroth via acmurthy)
+
+ YARN-234. Added support for process tree and resource calculator in MS Windows
+ to YARN. (Chris Nauroth via acmurthy)
+
+ YARN-259. Fix LocalDirsHandlerService to use Path rather than URIs. (Xuan
+ Gong via acmurthy)
+
+ YARN-316. YARN container launch may exceed maximum Windows command line
+ length due to long classpath. (Chris Nauroth via suresh)
+
+ YARN-359. Fixing commands for container signalling in Windows. (Chris Nauroth
+ via vinodkv)
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd Wed Mar 6 19:15:18 2013
@@ -0,0 +1,47 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+setlocal enabledelayedexpansion
+
+echo starting yarn daemons
+
+if not defined HADOOP_BIN_PATH (
+ set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+ set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+ set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %*
+if "%1" == "--config" (
+ shift
+ shift
+)
+
+@rem start resourceManager
+start "Apache Hadoop Distribution" yarn resourcemanager
+@rem start nodeManager
+start "Apache Hadoop Distribution" yarn nodemanager
+@rem start proxyserver
+@rem start "Apache Hadoop Distribution" yarn proxyserver
+
+endlocal
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd Wed Mar 6 19:15:18 2013
@@ -0,0 +1,47 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+setlocal enabledelayedexpansion
+
+echo stopping yarn daemons
+
+if not defined HADOOP_BIN_PATH (
+ set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+ set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+ set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %*
+if "%1" == "--config" (
+ shift
+ shift
+)
+
+@rem stop resourceManager
+Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - yarn resourcemanager"
+@rem stop nodeManager
+Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - yarn nodemanager"
+@rem stop proxy server
+Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - yarn proxyserver"
+
+endlocal
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn Wed Mar 6 19:15:18 2013
@@ -72,11 +72,6 @@ function print_usage(){
echo "Most commands print help when invoked w/o parameters."
}
-cygwin=false
-case "`uname`" in
-CYGWIN*) cygwin=true;;
-esac
-
# if no args specified, show usage
if [ $# = 0 ]; then
print_usage
@@ -177,9 +172,6 @@ unset IFS
# figure out which class to run
if [ "$COMMAND" = "classpath" ] ; then
- if $cygwin; then
- CLASSPATH=`cygpath -p -w "$CLASSPATH"`
- fi
echo $CLASSPATH
exit
elif [ "$COMMAND" = "rmadmin" ] ; then
@@ -227,19 +219,6 @@ else
CLASS=$COMMAND
fi
-# cygwin path translation
-if $cygwin; then
- CLASSPATH=`cygpath -p -w "$CLASSPATH"`
- HADOOP_YARN_HOME=`cygpath -w "$HADOOP_YARN_HOME"`
- YARN_LOG_DIR=`cygpath -w "$YARN_LOG_DIR"`
- TOOL_PATH=`cygpath -p -w "$TOOL_PATH"`
-fi
-
-# cygwin path translation
-if $cygwin; then
- JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
-fi
-
YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd Wed Mar 6 19:15:18 2013
@@ -0,0 +1,72 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem included in all the hdfs scripts with source command
+@rem should not be executed directly
+
+if not defined HADOOP_BIN_PATH (
+ set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+ set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+ set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+if exist %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd (
+ call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %*
+) else if exist %HADOOP_COMMON_HOME%\libexec\hadoop-config.cmd (
+ call %HADOOP_COMMON_HOME%\libexec\hadoop-config.cmd %*
+) else if exist %HADOOP_HOME%\libexec\hadoop-config.cmd (
+ call %HADOOP_HOME%\libexec\hadoop-config.cmd %*
+) else (
+ echo Hadoop common not found.
+)
+
+@rem
+@rem Allow alternate conf dir location.
+@rem
+
+if "%1" == "--config" (
+ shift
+ set YARN_CONF_DIR=%2
+ shift
+)
+
+if not defined YARN_CONF_DIR (
+ if not defined HADOOP_CONF_DIR (
+ set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf
+ ) else (
+ set YARN_CONF_DIR=%HADOOP_CONF_DIR%
+ )
+)
+
+@rem
+@rem check to see it is specified whether to use the slaves or the
+@rem masters file
+@rem
+
+if "%1" == "--hosts" (
+ set YARN_SLAVES=%YARN_CONF_DIR%\%2
+ shift
+ shift
+)
+
+:eof
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd Wed Mar 6 19:15:18 2013
@@ -0,0 +1,250 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem The Hadoop command script
+@rem
+@rem Environment Variables
+@rem
+@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME.
+@rem
+@rem YARN_CLASSPATH Extra Java CLASSPATH entries.
+@rem
+@rem YARN_HEAPSIZE The maximum amount of heap to use, in MB.
+@rem Default is 1000.
+@rem
+@rem YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command
+@rem eg YARN_NODEMANAGER_HEAPSIZE sets the heap
+@rem size for the NodeManager. If you set the
+@rem heap size in YARN_{COMMAND}_OPTS or YARN_OPTS
+@rem they take precedence.
+@rem
+@rem YARN_OPTS Extra Java runtime options.
+@rem
+@rem YARN_CLIENT_OPTS when the respective command is run.
+@rem YARN_{COMMAND}_OPTS etc YARN_NODEMANAGER_OPTS applies to NodeManager
+@rem for e.g. YARN_CLIENT_OPTS applies to
+@rem more than one command (fs, dfs, fsck,
+@rem dfsadmin etc)
+@rem
+@rem YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf.
+@rem
+@rem YARN_ROOT_LOGGER The root appender. Default is INFO,console
+@rem
+
+setlocal enabledelayedexpansion
+
+if not defined HADOOP_BIN_PATH (
+ set HADOOP_BIN_PATH=%~dp0
+)
+
+if "%HADOOP_BIN_PATH:~-1%" == "\" (
+ set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1%
+)
+
+set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec
+if not defined HADOOP_LIBEXEC_DIR (
+ set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR%
+)
+
+call %DEFAULT_LIBEXEC_DIR%\yarn-config.cmd %*
+if "%1" == "--config" (
+ shift
+ shift
+)
+
+:main
+ if exist %YARN_CONF_DIR%\yarn-env.cmd (
+ call %YARN_CONF_DIR%\yarn-env.cmd
+ )
+
+ set yarn-command=%1
+ call :make_command_arguments %*
+
+ if not defined yarn-command (
+ goto print_usage
+ )
+
+ @rem JAVA and JAVA_HEAP_MAX and set in hadoop-config.cmd
+
+ if defined YARN_HEAPSIZE (
+ @rem echo run with Java heapsize %YARN_HEAPSIZE%
+ set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
+ )
+
+ @rem CLASSPATH initially contains HADOOP_CONF_DIR & YARN_CONF_DIR
+ if not defined HADOOP_CONF_DIR (
+ echo No HADOOP_CONF_DIR set.
+ echo Please specify it either in yarn-env.cmd or in the environment.
+ goto :eof
+ )
+
+ set CLASSPATH=%HADOOP_CONF_DIR%;%YARN_CONF_DIR%;%CLASSPATH%
+
+ @rem for developers, add Hadoop classes to CLASSPATH
+ if exist %HADOOP_YARN_HOME%\yarn-api\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-api\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\yarn-common\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-common\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\yarn-mapreduce\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-mapreduce\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\yarn-master-worker\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-master-worker\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-nodemanager\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-nodemanager\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-common\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-common\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\build\test\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes
+ )
+
+ if exist %HADOOP_YARN_HOME%\build\tools (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\tools
+ )
+
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\*
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
+
+ call :%yarn-command% %yarn-command-arguments%
+
+ set java_arguments=%JAVA_HEAP_MAX% %YARN_OPTS% -classpath %CLASSPATH% %CLASS% %yarn-command-arguments%
+ call %JAVA% %java_arguments%
+
+goto :eof
+
+:classpath
+ @echo %CLASSPATH%
+ goto :eof
+
+:rmadmin
+ set CLASS=org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+:application
+ set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+:node
+ set CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+:resourcemanager
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
+ set YARN_OPTS=%YARN_OPTS% %HADOOP_RESOURCEMANAGER_OPTS%
+ if defined YARN_RESOURCEMANAGER_HEAPSIZE (
+ set JAVA_HEAP_MAX=-Xmx%YARN_RESOURCEMANAGER_HEAPSIZE%m
+ )
+ goto :eof
+
+:nodemanager
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
+ set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS%
+ if defined YARN_NODEMANAGER_HEAPSIZE (
+ set JAVA_HEAP_MAX=-Xmx%YARN_NODEMANAGER_HEAPSIZE%m
+ )
+ goto :eof
+
+:proxyserver
+ set CLASS=org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer
+ set YARN_OPTS=%YARN_OPTS% %HADOOP_PROXYSERVER_OPTS%
+ if defined YARN_PROXYSERVER_HEAPSIZE (
+ set JAVA_HEAP_MAX=-Xmx%YARN_PROXYSERVER_HEAPSIZE%m
+ )
+ goto :eof
+
+:version
+ set CLASS=org.apache.hadoop.util.VersionInfo
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+:jar
+ set CLASS=org.apache.hadoop.util.RunJar
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+:logs
+ set CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+:daemonlog
+ set CLASS=org.apache.hadoop.log.LogLevel
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
+@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
+:make_command_arguments
+ if "%1" == "--config" (
+ shift
+ shift
+ )
+ if [%2] == [] goto :eof
+ shift
+ set _yarnarguments=
+ :MakeCmdArgsLoop
+ if [%1]==[] goto :EndLoop
+
+ if not defined _yarnarguments (
+ set _yarnarguments=%1
+ ) else (
+ set _yarnarguments=!_yarnarguments! %1
+ )
+ shift
+ goto :MakeCmdArgsLoop
+ :EndLoop
+ set yarn-command-arguments=%_yarnarguments%
+ goto :eof
+
+:print_usage
+ @echo Usage: yarn [--config confdir] COMMAND
+ @echo where COMMAND is one of:
+ @echo resourcemanager run the ResourceManager
+ @echo nodemanager run a nodemanager on each slave
+ @echo historyserver run job history servers as a standalone daemon
+ @echo rmadmin admin tools
+ @echo version print the version
+ @echo jar ^<jar^> run a jar file
+ @echo application prints application(s) report/kill application
+ @echo node prints node report(s)
+ @echo logs dump container logs
+ @echo classpath prints the class path needed to get the
+ @echo Hadoop jar and the required libraries
+ @echo daemonlog get/set the log level for each daemon
+ @echo or
+ @echo CLASSNAME run the class named CLASSNAME
+ @echo Most commands print help when invoked w/o parameters.
+
+endlocal
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd Wed Mar 6 19:15:18 2013
@@ -0,0 +1,60 @@
+@echo off
+@rem Licensed to the Apache Software Foundation (ASF) under one or more
+@rem contributor license agreements. See the NOTICE file distributed with
+@rem this work for additional information regarding copyright ownership.
+@rem The ASF licenses this file to You under the Apache License, Version 2.0
+@rem (the "License"); you may not use this file except in compliance with
+@rem the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem User for YARN daemons
+if not defined HADOOP_YARN_USER (
+ set HADOOP_YARN_USER=%yarn%
+)
+
+if not defined YARN_CONF_DIR (
+ set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf
+)
+
+if defined YARN_HEAPSIZE (
+ @rem echo run with Java heapsize %YARN_HEAPSIZE%
+ set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m
+)
+
+if not defined YARN_LOG_DIR (
+ set YARN_LOG_DIR=%HADOOP_YARN_HOME%\logs
+)
+
+if not defined YARN_LOGFILE (
+ set YARN_LOGFILE=yarn.log
+)
+
+@rem default policy file for service-level authorization
+if not defined YARN_POLICYFILE (
+ set YARN_POLICYFILE=hadoop-policy.xml
+)
+
+if not defined YARN_ROOT_LOGGER (
+ set YARN_ROOT_LOGGER=INFO,console
+)
+
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.dir=%YARN_LOG_DIR%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.log.file=%YARN_LOGFILE%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.id.str=%YARN_IDENT_STRING%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.home.dir=%HADOOP_YARN_HOME%
+set YARN_OPTS=%YARN_OPTS% -Dhadoop.root.logger=%YARN_ROOT_LOGGER%
+set YARN_OPTS=%YARN_OPTS% -Dyarn.root.logger=%YARN_ROOT_LOGGER%
+if defined JAVA_LIBRARY_PATH (
+ set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
+)
+set YARN_OPTS=%YARN_OPTS% -Dyarn.policy.file=%YARN_POLICYFILE%
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Wed Mar 6 19:15:18 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
/**
* This is the API for the applications comprising of constants that YARN sets
@@ -192,7 +193,11 @@ public interface ApplicationConstants {
}
public String $() {
- return "$" + variable;
+ if (Shell.WINDOWS) {
+ return "%" + variable + "%";
+ } else {
+ return "$" + variable;
+ }
}
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml Wed Mar 6 19:15:18 2013
@@ -32,6 +32,28 @@
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
</modules>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <environmentVariables>
+ <!-- HADOOP_HOME required for tests on Windows to find winutils -->
+ <HADOOP_HOME>${basedir}/../../../../hadoop-common-project/hadoop-common/target</HADOOP_HOME>
+ </environmentVariables>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.test.TimedOutTestsListener</value>
+ </property>
+ </properties>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
<profiles>
<profile>
<id>clover</id>
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Wed Mar 6 19:15:18 2013
@@ -36,6 +36,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.StringUtils;
@@ -59,32 +60,30 @@ public class ProcfsBasedProcessTree exte
public static final String PROCFS_STAT_FILE = "stat";
public static final String PROCFS_CMDLINE_FILE = "cmdline";
public static final long PAGE_SIZE;
- static {
- ShellCommandExecutor shellExecutor =
- new ShellCommandExecutor(new String[]{"getconf", "PAGESIZE"});
- long pageSize = -1;
- try {
- shellExecutor.execute();
- pageSize = Long.parseLong(shellExecutor.getOutput().replace("\n", ""));
- } catch (IOException e) {
- LOG.error(StringUtils.stringifyException(e));
- } finally {
- PAGE_SIZE = pageSize;
- }
- }
public static final long JIFFY_LENGTH_IN_MILLIS; // in millisecond
+
static {
- ShellCommandExecutor shellExecutor =
- new ShellCommandExecutor(new String[]{"getconf", "CLK_TCK"});
long jiffiesPerSecond = -1;
+ long pageSize = -1;
try {
- shellExecutor.execute();
- jiffiesPerSecond = Long.parseLong(shellExecutor.getOutput().replace("\n", ""));
+ if(Shell.LINUX) {
+ ShellCommandExecutor shellExecutorClk = new ShellCommandExecutor(
+ new String[] { "getconf", "CLK_TCK" });
+ shellExecutorClk.execute();
+ jiffiesPerSecond = Long.parseLong(shellExecutorClk.getOutput().replace("\n", ""));
+
+ ShellCommandExecutor shellExecutorPage = new ShellCommandExecutor(
+ new String[] { "getconf", "PAGESIZE" });
+ shellExecutorPage.execute();
+ pageSize = Long.parseLong(shellExecutorPage.getOutput().replace("\n", ""));
+
+ }
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
} finally {
JIFFY_LENGTH_IN_MILLIS = jiffiesPerSecond != -1 ?
Math.round(1000D / jiffiesPerSecond) : -1;
+ PAGE_SIZE = pageSize;
}
}
@@ -126,8 +125,7 @@ public class ProcfsBasedProcessTree exte
*/
public static boolean isAvailable() {
try {
- String osName = System.getProperty("os.name");
- if (!osName.startsWith("Linux")) {
+ if (!Shell.LINUX) {
LOG.info("ProcfsBasedProcessTree currently is supported only on "
+ "Linux.");
return false;
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java Wed Mar 6 19:15:18 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
/**
* Plugin to calculate resource information on the system.
@@ -31,6 +32,18 @@ import org.apache.hadoop.util.Reflection
@InterfaceAudience.Private
@InterfaceStability.Unstable
public abstract class ResourceCalculatorPlugin extends Configured {
+
+ protected String processPid = null;
+
+ /**
+ * set the pid of the process for which <code>getProcResourceValues</code>
+ * will be invoked
+ *
+ * @param pid
+ */
+ public void setProcessPid(String pid) {
+ processPid = pid;
+ }
/**
* Obtain the total size of the virtual memory present in the system.
@@ -109,10 +122,12 @@ public abstract class ResourceCalculator
// No class given, try a os specific class
try {
- String osName = System.getProperty("os.name");
- if (osName.startsWith("Linux")) {
+ if (Shell.LINUX) {
return new LinuxResourceCalculatorPlugin();
}
+ if (Shell.WINDOWS) {
+ return new WindowsResourceCalculatorPlugin();
+ }
} catch (SecurityException se) {
// Failed to get Operating System name.
return null;
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java Wed Mar 6 19:15:18 2013
@@ -145,14 +145,11 @@ public abstract class ResourceCalculator
}
// No class given, try a os specific class
- try {
- String osName = System.getProperty("os.name");
- if (osName.startsWith("Linux")) {
- return new ProcfsBasedProcessTree(pid);
- }
- } catch (SecurityException se) {
- // Failed to get Operating System name.
- return null;
+ if (ProcfsBasedProcessTree.isAvailable()) {
+ return new ProcfsBasedProcessTree(pid);
+ }
+ if (WindowsBasedProcessTree.isAvailable()) {
+ return new WindowsBasedProcessTree(pid);
}
// Not supported on this system.
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java Wed Mar 6 19:15:18 2013
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.util.StringUtils;
+
+
+public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
+
+ static final Log LOG = LogFactory
+ .getLog(WindowsBasedProcessTree.class);
+
+ static class ProcessInfo {
+ String pid; // process pid
+ long vmem; // virtual memory
+ long workingSet; // working set, RAM used
+ long cpuTimeMs; // total cpuTime in millisec
+ long cpuTimeMsDelta; // delta of cpuTime since last update
+ int age = 1;
+ }
+
+ private String taskProcessId = null;
+ private long cpuTimeMs = 0;
+ private Map<String, ProcessInfo> processTree =
+ new HashMap<String, ProcessInfo>();
+
+ public static boolean isAvailable() {
+ if (Shell.WINDOWS) {
+ ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
+ new String[] { Shell.WINUTILS, "help" });
+ try {
+ shellExecutor.execute();
+ } catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ } finally {
+ String output = shellExecutor.getOutput();
+ if (output != null &&
+ output.contains("Prints to stdout a list of processes in the task")) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public WindowsBasedProcessTree(String pid) {
+ super(pid);
+ taskProcessId = pid;
+ }
+
+ // helper method to override while testing
+ String getAllProcessInfoFromShell() {
+ ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
+ new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
+ try {
+ shellExecutor.execute();
+ return shellExecutor.getOutput();
+ } catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ }
+ return null;
+ }
+
+ /**
+ * Parses string of process info lines into ProcessInfo objects
+ * @param processesInfoStr
+ * @return Map of pid string to ProcessInfo objects
+ */
+ Map<String, ProcessInfo> createProcessInfo(String processesInfoStr) {
+ String[] processesStr = processesInfoStr.split("\r\n");
+ Map<String, ProcessInfo> allProcs = new HashMap<String, ProcessInfo>();
+ final int procInfoSplitCount = 4;
+ for (String processStr : processesStr) {
+ if (processStr != null) {
+ String[] procInfo = processStr.split(",");
+ if (procInfo.length == procInfoSplitCount) {
+ try {
+ ProcessInfo pInfo = new ProcessInfo();
+ pInfo.pid = procInfo[0];
+ pInfo.vmem = Long.parseLong(procInfo[1]);
+ pInfo.workingSet = Long.parseLong(procInfo[2]);
+ pInfo.cpuTimeMs = Long.parseLong(procInfo[3]);
+ allProcs.put(pInfo.pid, pInfo);
+ } catch (NumberFormatException nfe) {
+ LOG.debug("Error parsing procInfo." + nfe);
+ }
+ } else {
+ LOG.debug("Expected split length of proc info to be "
+ + procInfoSplitCount + ". Got " + procInfo.length);
+ }
+ }
+ }
+ return allProcs;
+ }
+
+ @Override
+ public void updateProcessTree() {
+ if(taskProcessId != null) {
+ // taskProcessId can be null in some tests
+ String processesInfoStr = getAllProcessInfoFromShell();
+ if (processesInfoStr != null && processesInfoStr.length() > 0) {
+ Map<String, ProcessInfo> allProcessInfo = createProcessInfo(processesInfoStr);
+
+ for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet()) {
+ String pid = entry.getKey();
+ ProcessInfo pInfo = entry.getValue();
+ ProcessInfo oldInfo = processTree.get(pid);
+ if (oldInfo != null) {
+ // existing process, update age and replace value
+ pInfo.age += oldInfo.age;
+ // calculate the delta since the last refresh. totals are being kept
+ // in the WindowsBasedProcessTree object
+ pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs;
+ } else {
+ // new process. delta cpu == total cpu
+ pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs;
+ }
+ }
+ processTree.clear();
+ processTree = allProcessInfo;
+ } else {
+ // clearing process tree to mimic semantics of existing Procfs impl
+ processTree.clear();
+ }
+ }
+ }
+
+ @Override
+ public boolean checkPidPgrpidForMatch() {
+ // This is always true on Windows, because the pid doubles as a job object
+ // name for task management.
+ return true;
+ }
+
+ @Override
+ public String getProcessTreeDump() {
+ StringBuilder ret = new StringBuilder();
+ // The header.
+ ret.append(String.format("\t|- PID " + "CPU_TIME(MILLIS) "
+ + "VMEM(BYTES) WORKING_SET(BYTES)\n"));
+ for (ProcessInfo p : processTree.values()) {
+ if (p != null) {
+ ret.append(String.format("\t|- %s %d %d %d\n", p.pid,
+ p.cpuTimeMs, p.vmem, p.workingSet));
+ }
+ }
+ return ret.toString();
+ }
+
+ @Override
+ public long getCumulativeVmem(int olderThanAge) {
+ long total = 0;
+ for (ProcessInfo p : processTree.values()) {
+ if ((p != null) && (p.age > olderThanAge)) {
+ total += p.vmem;
+ }
+ }
+ return total;
+ }
+
+ @Override
+ public long getCumulativeRssmem(int olderThanAge) {
+ long total = 0;
+ for (ProcessInfo p : processTree.values()) {
+ if ((p != null) && (p.age > olderThanAge)) {
+ total += p.workingSet;
+ }
+ }
+ return total;
+ }
+
+ @Override
+ public long getCumulativeCpuTime() {
+ for (ProcessInfo p : processTree.values()) {
+ cpuTimeMs += p.cpuTimeMsDelta;
+ }
+ return cpuTimeMs;
+ }
+
+}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java Wed Mar 6 19:15:18 2013
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.util.StringUtils;
+
+public class WindowsResourceCalculatorPlugin extends ResourceCalculatorPlugin {
+
+ static final Log LOG = LogFactory
+ .getLog(WindowsResourceCalculatorPlugin.class);
+
+ long vmemSize;
+ long memSize;
+ long vmemAvailable;
+ long memAvailable;
+ int numProcessors;
+ long cpuFrequencyKhz;
+ long cumulativeCpuTimeMs;
+ float cpuUsage;
+
+ long lastRefreshTime;
+ private final int refreshIntervalMs = 1000;
+
+ WindowsBasedProcessTree pTree = null;
+
+ public WindowsResourceCalculatorPlugin() {
+ lastRefreshTime = 0;
+ reset();
+ }
+
+ void reset() {
+ vmemSize = -1;
+ memSize = -1;
+ vmemAvailable = -1;
+ memAvailable = -1;
+ numProcessors = -1;
+ cpuFrequencyKhz = -1;
+ cumulativeCpuTimeMs = -1;
+ cpuUsage = -1;
+ }
+
+ String getSystemInfoInfoFromShell() {
+ ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
+ new String[] { Shell.WINUTILS, "systeminfo" });
+ try {
+ shellExecutor.execute();
+ return shellExecutor.getOutput();
+ } catch (IOException e) {
+ LOG.error(StringUtils.stringifyException(e));
+ }
+ return null;
+ }
+
+ void refreshIfNeeded() {
+ long now = System.currentTimeMillis();
+ if (now - lastRefreshTime > refreshIntervalMs) {
+ long refreshInterval = now - lastRefreshTime;
+ lastRefreshTime = now;
+ long lastCumCpuTimeMs = cumulativeCpuTimeMs;
+ reset();
+ String sysInfoStr = getSystemInfoInfoFromShell();
+ if (sysInfoStr != null) {
+ final int sysInfoSplitCount = 7;
+ String[] sysInfo = sysInfoStr.substring(0, sysInfoStr.indexOf("\r\n"))
+ .split(",");
+ if (sysInfo.length == sysInfoSplitCount) {
+ try {
+ vmemSize = Long.parseLong(sysInfo[0]);
+ memSize = Long.parseLong(sysInfo[1]);
+ vmemAvailable = Long.parseLong(sysInfo[2]);
+ memAvailable = Long.parseLong(sysInfo[3]);
+ numProcessors = Integer.parseInt(sysInfo[4]);
+ cpuFrequencyKhz = Long.parseLong(sysInfo[5]);
+ cumulativeCpuTimeMs = Long.parseLong(sysInfo[6]);
+ if (lastCumCpuTimeMs != -1) {
+ cpuUsage = (cumulativeCpuTimeMs - lastCumCpuTimeMs)
+ / (refreshInterval * 1.0f);
+ }
+
+ } catch (NumberFormatException nfe) {
+ LOG.warn("Error parsing sysInfo." + nfe);
+ }
+ } else {
+ LOG.warn("Expected split length of sysInfo to be "
+ + sysInfoSplitCount + ". Got " + sysInfo.length);
+ }
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getVirtualMemorySize() {
+ refreshIfNeeded();
+ return vmemSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getPhysicalMemorySize() {
+ refreshIfNeeded();
+ return memSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getAvailableVirtualMemorySize() {
+ refreshIfNeeded();
+ return vmemAvailable;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getAvailablePhysicalMemorySize() {
+ refreshIfNeeded();
+ return memAvailable;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int getNumProcessors() {
+ refreshIfNeeded();
+ return numProcessors;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getCpuFrequency() {
+ refreshIfNeeded();
+ return -1;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getCumulativeCpuTime() {
+ refreshIfNeeded();
+ return cumulativeCpuTimeMs;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public float getCpuUsage() {
+ refreshIfNeeded();
+ return cpuUsage;
+ }
+}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java Wed Mar 6 19:15:18 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
@@ -104,17 +105,21 @@ public class TestProcfsBasedProcessTree
new Path(TEST_ROOT_DIR.getAbsolutePath()), true);
}
- @Test
+ @Test (timeout = 30000)
public void testProcessTree() throws Exception {
+ if (!Shell.LINUX) {
+ System.out
+ .println("ProcfsBasedProcessTree is not available on this system. Not testing");
+ return;
+
+ }
try {
- if (!ProcfsBasedProcessTree.isAvailable()) {
- System.out
- .println("ProcfsBasedProcessTree is not available on this system. Not testing");
- return;
- }
+ Assert.assertTrue(ProcfsBasedProcessTree.isAvailable());
} catch (Exception e) {
LOG.info(StringUtils.stringifyException(e));
+ Assert.assertTrue("ProcfsBaseProcessTree should be available on Linux",
+ false);
return;
}
// create shell script
@@ -328,7 +333,7 @@ public class TestProcfsBasedProcessTree
* @throws IOException if there was a problem setting up the
* fake procfs directories or files.
*/
- @Test
+ @Test (timeout = 30000)
public void testCpuAndMemoryForProcessTree() throws IOException {
// test processes
@@ -402,7 +407,7 @@ public class TestProcfsBasedProcessTree
* @throws IOException if there was a problem setting up the
* fake procfs directories or files.
*/
- @Test
+ @Test (timeout = 30000)
public void testMemForOlderProcesses() throws IOException {
// initial list of processes
String[] pids = { "100", "200", "300", "400" };
@@ -509,7 +514,7 @@ public class TestProcfsBasedProcessTree
* @throws IOException if there was a problem setting up the
* fake procfs directories or files.
*/
- @Test
+ @Test (timeout = 30000)
public void testDestroyProcessTree() throws IOException {
// test process
String pid = "100";
@@ -535,7 +540,7 @@ public class TestProcfsBasedProcessTree
*
* @throws IOException
*/
- @Test
+ @Test (timeout = 30000)
public void testProcessTreeDump()
throws IOException {
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java Wed Mar 6 19:15:18 2013
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+public class TestWindowsBasedProcessTree extends TestCase {
+ private static final Log LOG = LogFactory
+ .getLog(TestWindowsBasedProcessTree.class);
+
+ class WindowsBasedProcessTreeTester extends WindowsBasedProcessTree {
+ String infoStr = null;
+ public WindowsBasedProcessTreeTester(String pid) {
+ super(pid);
+ }
+ @Override
+ String getAllProcessInfoFromShell() {
+ return infoStr;
+ }
+ }
+
+ @Test (timeout = 30000)
+ public void testTree() {
+ if( !Shell.WINDOWS) {
+ LOG.info("Platform not Windows. Not testing");
+ return;
+ }
+ assertTrue("WindowsBasedProcessTree should be available on Windows",
+ WindowsBasedProcessTree.isAvailable());
+
+
+ WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1");
+ pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n";
+ pTree.updateProcessTree();
+ assertTrue(pTree.getCumulativeVmem() == 2048);
+ assertTrue(pTree.getCumulativeVmem(0) == 2048);
+ assertTrue(pTree.getCumulativeRssmem() == 2048);
+ assertTrue(pTree.getCumulativeRssmem(0) == 2048);
+ assertTrue(pTree.getCumulativeCpuTime() == 1000);
+
+ pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n";
+ pTree.updateProcessTree();
+ assertTrue(pTree.getCumulativeVmem() == 3072);
+ assertTrue(pTree.getCumulativeVmem(1) == 2048);
+ assertTrue(pTree.getCumulativeRssmem() == 3072);
+ assertTrue(pTree.getCumulativeRssmem(1) == 2048);
+ assertTrue(pTree.getCumulativeCpuTime() == 3000);
+
+ pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
+ pTree.updateProcessTree();
+ assertTrue(pTree.getCumulativeVmem() == 2048);
+ assertTrue(pTree.getCumulativeVmem(2) == 2048);
+ assertTrue(pTree.getCumulativeRssmem() == 2048);
+ assertTrue(pTree.getCumulativeRssmem(2) == 2048);
+ assertTrue(pTree.getCumulativeCpuTime() == 4000);
+ }
+}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java?rev=1453486&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java Wed Mar 6 19:15:18 2013
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+public class TestWindowsResourceCalculatorPlugin extends TestCase {
+
+
+ class WindowsResourceCalculatorPluginTester extends WindowsResourceCalculatorPlugin {
+ private String infoStr = null;
+ @Override
+ String getSystemInfoInfoFromShell() {
+ return infoStr;
+ }
+ }
+
+ @Test (timeout = 30000)
+ public void testParseSystemInfoString() {
+ WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
+ // info str derived from windows shell command has \r\n termination
+ tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n";
+ // call a method to refresh values
+ tester.getAvailablePhysicalMemorySize();
+ // verify information has been refreshed
+ assertTrue(tester.vmemSize == 17177038848L);
+ assertTrue(tester.memSize == 8589467648L);
+ assertTrue(tester.vmemAvailable == 15232745472L);
+ assertTrue(tester.memAvailable == 6400417792L);
+ assertTrue(tester.numProcessors == 1);
+ assertTrue(tester.cpuFrequencyKhz == 2805000L);
+ assertTrue(tester.cumulativeCpuTimeMs == 6261812L);
+ assertTrue(tester.cpuUsage == -1);
+ }
+
+ @Test (timeout = 20000)
+ public void testRefreshAndCpuUsage() throws InterruptedException {
+ WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
+ // info str derived from windows shell command has \r\n termination
+ tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n";
+ tester.getAvailablePhysicalMemorySize();
+ // verify information has been refreshed
+ assertTrue(tester.memAvailable == 6400417792L);
+ assertTrue(tester.cpuUsage == -1);
+
+ tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6261812\r\n";
+ tester.getAvailablePhysicalMemorySize();
+ // verify information has not been refreshed
+ assertTrue(tester.memAvailable == 6400417792L);
+ assertTrue(tester.cpuUsage == -1);
+
+ Thread.sleep(1500);
+ tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6286812\r\n";
+ tester.getAvailablePhysicalMemorySize();
+ // verify information has been refreshed
+ assertTrue(tester.memAvailable == 5400417792L);
+ assertTrue(tester.cpuUsage >= 0.1);
+ }
+
+ @Test (timeout = 20000)
+ public void testErrorInGetSystemInfo() {
+ WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
+ // info str derived from windows shell command has \r\n termination
+ tester.infoStr = null;
+ // call a method to refresh values
+ tester.getAvailablePhysicalMemorySize();
+ }
+
+}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Wed Mar 6 19:15:18 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.util.Shell.Shel
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
+import org.apache.hadoop.util.Shell;
public abstract class ContainerExecutor implements Configurable {
@@ -182,6 +183,33 @@ public abstract class ContainerExecutor
readLock.unlock();
}
}
+
+ /** Return a command to execute the given command in OS shell.
+ * On Windows, the passed in groupId can be used to launch
+ * and associate the given groupId in a process group. On
+ * non-Windows, groupId is ignored. */
+ protected static String[] getRunCommand(String command,
+ String groupId) {
+ if (Shell.WINDOWS) {
+ return new String[] { Shell.WINUTILS, "task", "create", groupId,
+ "cmd /c " + command };
+ } else {
+ return new String[] { "bash", "-c", command };
+ }
+ }
+
+ /** Return a command for determining if process with specified pid is alive. */
+ protected static String[] getCheckProcessIsAliveCommand(String pid) {
+ return Shell.WINDOWS ?
+ new String[] { Shell.WINUTILS, "task", "isAlive", pid } :
+ new String[] { "kill", "-0", pid };
+ }
+
+ /** Return a command to send a signal to a given pid */
+ protected static String[] getSignalKillCommand(int code, String pid) {
+ return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid } :
+ new String[] { "kill", "-" + code, pid };
+ }
/**
* Is the container still active?
@@ -253,6 +281,9 @@ public abstract class ContainerExecutor
public static final boolean isSetsidAvailable = isSetsidSupported();
private static boolean isSetsidSupported() {
+ if (Shell.WINDOWS) {
+ return true;
+ }
ShellCommandExecutor shexec = null;
boolean setsidSupported = true;
try {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Wed Mar 6 19:15:18 2013
@@ -37,6 +37,8 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -53,10 +55,9 @@ public class DefaultContainerExecutor ex
private static final Log LOG = LogFactory
.getLog(DefaultContainerExecutor.class);
- private final FileContext lfs;
+ private static final int WIN_MAX_PATH = 260;
- private static final String WRAPPER_LAUNCH_SCRIPT =
- "default_container_executor.sh";
+ private final FileContext lfs;
public DefaultContainerExecutor() {
try {
@@ -145,15 +146,24 @@ public class DefaultContainerExecutor ex
lfs.util().copy(nmPrivateTokensPath, tokenDst);
// Create new local launch wrapper script
- Path wrapperScriptDst = new Path(containerWorkDir, WRAPPER_LAUNCH_SCRIPT);
- DataOutputStream wrapperScriptOutStream =
- lfs.create(wrapperScriptDst,
- EnumSet.of(CREATE, OVERWRITE));
+ LocalWrapperScriptBuilder sb = Shell.WINDOWS ?
+ new WindowsLocalWrapperScriptBuilder(containerIdStr, containerWorkDir) :
+ new UnixLocalWrapperScriptBuilder(containerWorkDir);
+
+ // Fail fast if attempting to launch the wrapper script would fail due to
+ // Windows path length limitation.
+ if (Shell.WINDOWS &&
+ sb.getWrapperScriptPath().toString().length() > WIN_MAX_PATH) {
+ throw new IOException(String.format(
+ "Cannot launch container using script at path %s, because it exceeds " +
+ "the maximum supported path length of %d characters. Consider " +
+ "configuring shorter directories in %s.", sb.getWrapperScriptPath(),
+ WIN_MAX_PATH, YarnConfiguration.NM_LOCAL_DIRS));
+ }
Path pidFile = getPidFilePath(containerId);
if (pidFile != null) {
- writeLocalWrapperScript(wrapperScriptOutStream, launchDst.toUri()
- .getPath().toString(), pidFile.toString());
+ sb.writeLocalWrapperScript(launchDst, pidFile);
} else {
LOG.info("Container " + containerIdStr
+ " was marked as inactive. Returning terminated error");
@@ -166,12 +176,13 @@ public class DefaultContainerExecutor ex
try {
lfs.setPermission(launchDst,
ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
- lfs.setPermission(wrapperScriptDst,
+ lfs.setPermission(sb.getWrapperScriptPath(),
ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
// Setup command to run
- String[] command = {"bash",
- wrapperScriptDst.toUri().getPath().toString()};
+ String[] command = getRunCommand(sb.getWrapperScriptPath().toString(),
+ containerIdStr);
+
LOG.info("launchContainer: " + Arrays.toString(command));
shExec = new ShellCommandExecutor(
command,
@@ -202,28 +213,85 @@ public class DefaultContainerExecutor ex
return 0;
}
- private void writeLocalWrapperScript(DataOutputStream out,
- String launchScriptDst, String pidFilePath) throws IOException {
- // We need to do a move as writing to a file is not atomic
- // Process reading a file being written to may get garbled data
- // hence write pid to tmp file first followed by a mv
- StringBuilder sb = new StringBuilder("#!/bin/bash\n\n");
- sb.append("echo $$ > " + pidFilePath + ".tmp\n");
- sb.append("/bin/mv -f " + pidFilePath + ".tmp " + pidFilePath + "\n");
- sb.append(ContainerExecutor.isSetsidAvailable? "exec setsid" : "exec");
- sb.append(" /bin/bash ");
- sb.append("\"");
- sb.append(launchScriptDst);
- sb.append("\"\n");
- PrintStream pout = null;
- try {
- pout = new PrintStream(out);
- pout.append(sb);
- } finally {
- if (out != null) {
- out.close();
+ private abstract class LocalWrapperScriptBuilder {
+
+ private final Path wrapperScriptPath;
+
+ public Path getWrapperScriptPath() {
+ return wrapperScriptPath;
+ }
+
+ public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOException {
+ DataOutputStream out = null;
+ PrintStream pout = null;
+
+ try {
+ out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE));
+ pout = new PrintStream(out);
+ writeLocalWrapperScript(launchDst, pidFile, pout);
+ } finally {
+ IOUtils.cleanup(LOG, pout, out);
}
}
+
+ protected abstract void writeLocalWrapperScript(Path launchDst, Path pidFile,
+ PrintStream pout);
+
+ protected LocalWrapperScriptBuilder(Path wrapperScriptPath) {
+ this.wrapperScriptPath = wrapperScriptPath;
+ }
+ }
+
+ private final class UnixLocalWrapperScriptBuilder
+ extends LocalWrapperScriptBuilder {
+
+ public UnixLocalWrapperScriptBuilder(Path containerWorkDir) {
+ super(new Path(containerWorkDir, "default_container_executor.sh"));
+ }
+
+ @Override
+ public void writeLocalWrapperScript(Path launchDst, Path pidFile,
+ PrintStream pout) {
+
+ // We need to do a move as writing to a file is not atomic
+ // Process reading a file being written to may get garbled data
+ // hence write pid to tmp file first followed by a mv
+ pout.println("#!/bin/bash");
+ pout.println();
+ pout.println("echo $$ > " + pidFile.toString() + ".tmp");
+ pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile);
+ String exec = ContainerExecutor.isSetsidAvailable? "exec setsid" : "exec";
+ pout.println(exec + " /bin/bash -c \"" +
+ launchDst.toUri().getPath().toString() + "\"");
+ }
+ }
+
+ private final class WindowsLocalWrapperScriptBuilder
+ extends LocalWrapperScriptBuilder {
+
+ private final String containerIdStr;
+
+ public WindowsLocalWrapperScriptBuilder(String containerIdStr,
+ Path containerWorkDir) {
+
+ super(new Path(containerWorkDir, "default_container_executor.cmd"));
+ this.containerIdStr = containerIdStr;
+ }
+
+ @Override
+ public void writeLocalWrapperScript(Path launchDst, Path pidFile,
+ PrintStream pout) {
+
+ // On Windows, the pid is the container ID, so that it can also serve as
+ // the name of the job object created by winutils for task management.
+ // Write to temp file followed by atomic move.
+ String normalizedPidFile = new File(pidFile.toString()).getPath();
+ pout.println("@echo " + containerIdStr + " > " + normalizedPidFile +
+ ".tmp");
+ pout.println("@move /Y " + normalizedPidFile + ".tmp " +
+ normalizedPidFile);
+ pout.println("@call " + launchDst.toString());
+ }
}
@Override
@@ -234,17 +302,13 @@ public class DefaultContainerExecutor ex
: pid;
LOG.debug("Sending signal " + signal.getValue() + " to pid " + sigpid
+ " as user " + user);
- try {
- sendSignal(sigpid, Signal.NULL);
- } catch (ExitCodeException e) {
+ if (!containerIsAlive(sigpid)) {
return false;
}
try {
- sendSignal(sigpid, signal);
+ killContainer(sigpid, signal);
} catch (IOException e) {
- try {
- sendSignal(sigpid, Signal.NULL);
- } catch (IOException ignore) {
+ if (!containerIsAlive(sigpid)) {
return false;
}
throw e;
@@ -253,17 +317,33 @@ public class DefaultContainerExecutor ex
}
/**
+ * Returns true if the process with the specified pid is alive.
+ *
+ * @param pid String pid
+ * @return boolean true if the process is alive
+ */
+ private boolean containerIsAlive(String pid) throws IOException {
+ try {
+ new ShellCommandExecutor(getCheckProcessIsAliveCommand(pid)).execute();
+ // successful execution means process is alive
+ return true;
+ }
+ catch (ExitCodeException e) {
+ // failure (non-zero exit code) means process is not alive
+ return false;
+ }
+ }
+
+ /**
* Send a specified signal to the specified pid
*
* @param pid the pid of the process [group] to signal.
* @param signal signal to send
* (for logging).
*/
- protected void sendSignal(String pid, Signal signal) throws IOException {
- ShellCommandExecutor shexec = null;
- String[] arg = { "kill", "-" + signal.getValue(), pid };
- shexec = new ShellCommandExecutor(arg);
- shexec.execute();
+ private void killContainer(String pid, Signal signal) throws IOException {
+ new ShellCommandExecutor(getSignalKillCommand(signal.getValue(), pid))
+ .execute();
}
@Override
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java Wed Mar 6 19:15:18 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.no
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Timer;
@@ -305,7 +304,7 @@ public class LocalDirsHandlerService ext
ArrayList<String> validPaths = new ArrayList<String>();
for (int i = 0; i < paths.length; ++i) {
try {
- URI uriPath = new URI(paths[i]);
+ URI uriPath = (new Path(paths[i])).toUri();
if (uriPath.getScheme() == null
|| uriPath.getScheme().equals(FILE_SCHEME)) {
validPaths.add(uriPath.getPath());
@@ -316,7 +315,7 @@ public class LocalDirsHandlerService ext
+ " is not a valid path. Path should be with " + FILE_SCHEME
+ " scheme or without scheme");
}
- } catch (URISyntaxException e) {
+ } catch (IllegalArgumentException e) {
LOG.warn(e.getMessage());
throw new YarnException(paths[i]
+ " is not a valid path. Path should be with " + FILE_SCHEME
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Wed Mar 6 19:15:18 2013
@@ -23,6 +23,7 @@ import static org.apache.hadoop.fs.Creat
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.File;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
@@ -37,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
@@ -69,7 +71,8 @@ public class ContainerLaunch implements
private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
- public static final String CONTAINER_SCRIPT = "launch_container.sh";
+ public static final String CONTAINER_SCRIPT = Shell.WINDOWS ?
+ "launch_container.cmd" : "launch_container.sh";
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
@@ -130,7 +133,7 @@ public class ContainerLaunch implements
for (String str : command) {
// TODO: Should we instead work via symlinks without this grammar?
newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toUri().getPath()));
+ containerLogDir.toString()));
}
launchContext.setCommands(newCmds);
@@ -141,7 +144,7 @@ public class ContainerLaunch implements
entry.setValue(
value.replace(
ApplicationConstants.LOG_DIR_EXPANSION_VAR,
- containerLogDir.toUri().getPath())
+ containerLogDir.toString())
);
}
// /////////////////////////// End of variable expansion
@@ -411,28 +414,17 @@ public class ContainerLaunch implements
+ appIdStr;
}
- private static class ShellScriptBuilder {
-
- private final StringBuilder sb;
-
- public ShellScriptBuilder() {
- this(new StringBuilder("#!/bin/bash\n\n"));
- }
-
- protected ShellScriptBuilder(StringBuilder sb) {
- this.sb = sb;
- }
-
- public ShellScriptBuilder env(String key, String value) {
- line("export ", key, "=\"", value, "\"");
- return this;
- }
-
- public ShellScriptBuilder symlink(Path src, String dst) throws IOException {
- return symlink(src, new Path(dst));
- }
-
- public ShellScriptBuilder symlink(Path src, Path dst) throws IOException {
+ private static abstract class ShellScriptBuilder {
+
+ private static final String LINE_SEPARATOR =
+ System.getProperty("line.separator");
+ private final StringBuilder sb = new StringBuilder();
+
+ public abstract void command(List<String> command);
+
+ public abstract void env(String key, String value);
+
+ public final void symlink(Path src, Path dst) throws IOException {
if (!src.isAbsolute()) {
throw new IOException("Source must be absolute");
}
@@ -440,28 +432,89 @@ public class ContainerLaunch implements
throw new IOException("Destination must be relative");
}
if (dst.toUri().getPath().indexOf('/') != -1) {
- line("mkdir -p ", dst.getParent().toString());
+ mkdir(dst.getParent());
}
- line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
- return this;
+ link(src, dst);
}
-
- public void write(PrintStream out) throws IOException {
+
+ @Override
+ public String toString() {
+ return sb.toString();
+ }
+
+ public final void write(PrintStream out) throws IOException {
out.append(sb);
}
-
- public void line(String... command) {
+
+ protected final void line(String... command) {
for (String s : command) {
sb.append(s);
}
- sb.append("\n");
+ sb.append(LINE_SEPARATOR);
}
-
+
+ protected abstract void link(Path src, Path dst) throws IOException;
+
+ protected abstract void mkdir(Path path);
+ }
+
+ private static final class UnixShellScriptBuilder extends ShellScriptBuilder {
+
+ public UnixShellScriptBuilder(){
+ line("#!/bin/bash");
+ line();
+ }
+
@Override
- public String toString() {
- return sb.toString();
+ public void command(List<String> command) {
+ line("exec /bin/bash -c \"", StringUtils.join(" ", command), "\"");
+ }
+
+ @Override
+ public void env(String key, String value) {
+ line("export ", key, "=\"", value, "\"");
+ }
+
+ @Override
+ protected void link(Path src, Path dst) throws IOException {
+ line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
+ }
+
+ @Override
+ protected void mkdir(Path path) {
+ line("mkdir -p ", path.toString());
+ }
+ }
+
+ private static final class WindowsShellScriptBuilder
+ extends ShellScriptBuilder {
+
+ public WindowsShellScriptBuilder() {
+ line("@setlocal");
+ line();
+ }
+
+ @Override
+ public void command(List<String> command) {
+ line("@call ", StringUtils.join(" ", command));
+ }
+
+ @Override
+ public void env(String key, String value) {
+ line("@set ", key, "=", value);
+ }
+
+ @Override
+ protected void link(Path src, Path dst) throws IOException {
+ line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS,
+ new File(dst.toString()).getPath(),
+ new File(src.toUri().getPath()).getPath()));
}
+ @Override
+ protected void mkdir(Path path) {
+ line("@if not exist ", path.toString(), " mkdir ", path.toString());
+ }
}
private static void putEnvIfNotNull(
@@ -479,7 +532,7 @@ public class ContainerLaunch implements
}
public void sanitizeEnv(Map<String, String> environment,
- Path pwd, List<Path> appDirs) {
+ Path pwd, List<Path> appDirs) throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -513,6 +566,14 @@ public class ContainerLaunch implements
environment.put("JVM_PID", "$$");
}
+ // TODO: Remove Windows check and use this approach on all platforms after
+ // additional testing. See YARN-358.
+ if (Shell.WINDOWS) {
+ String inputClassPath = environment.get(Environment.CLASSPATH.name());
+ environment.put(Environment.CLASSPATH.name(),
+ FileUtil.createJarWithClassPath(inputClassPath, pwd));
+ }
+
/**
* Modifiable environment variables
*/
@@ -537,7 +598,8 @@ public class ContainerLaunch implements
Map<String,String> environment, Map<Path,List<String>> resources,
List<String> command)
throws IOException {
- ShellScriptBuilder sb = new ShellScriptBuilder();
+ ShellScriptBuilder sb = Shell.WINDOWS ? new WindowsShellScriptBuilder() :
+ new UnixShellScriptBuilder();
if (environment != null) {
for (Map.Entry<String,String> env : environment.entrySet()) {
sb.env(env.getKey().toString(), env.getValue().toString());
@@ -546,21 +608,13 @@ public class ContainerLaunch implements
if (resources != null) {
for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
for (String linkName : entry.getValue()) {
- sb.symlink(entry.getKey(), linkName);
+ sb.symlink(entry.getKey(), new Path(linkName));
}
}
}
- ArrayList<String> cmd = new ArrayList<String>(2 * command.size() + 5);
- cmd.add("exec /bin/bash ");
- cmd.add("-c ");
- cmd.add("\"");
- for (String cs : command) {
- cmd.add(cs.toString());
- cmd.add(" ");
- }
- cmd.add("\"");
- sb.line(cmd.toArray(new String[cmd.size()]));
+ sb.command(command);
+
PrintStream pout = null;
try {
pout = new PrintStream(out);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java Wed Mar 6 19:15:18 2013
@@ -25,6 +25,8 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Helper functionality to read the pid from a file.
@@ -62,14 +64,28 @@ public class ProcessIdFileReader {
}
String temp = line.trim();
if (!temp.isEmpty()) {
- try {
- Long pid = Long.valueOf(temp);
- if (pid > 0) {
+ if (Shell.WINDOWS) {
+ // On Windows, pid is expected to be a container ID, so find first
+ // line that parses successfully as a container ID.
+ try {
+ ConverterUtils.toContainerId(temp);
processId = temp;
break;
+ } catch (Exception e) {
+ // do nothing
+ }
+ }
+ else {
+ // Otherwise, find first line containing a numeric pid.
+ try {
+ Long pid = Long.valueOf(temp);
+ if (pid > 0) {
+ processId = temp;
+ break;
+ }
+ } catch (Exception e) {
+ // do nothing
}
- } catch (Exception e) {
- // do nothing
}
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java?rev=1453486&r1=1453485&r2=1453486&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java Wed Mar 6 19:15:18 2013
@@ -26,13 +26,14 @@ import java.io.PrintWriter;
import junit.framework.Assert;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.junit.Test;
public class TestProcessIdFileReader {
- @Test
+ @Test (timeout = 30000)
public void testNullPath() {
String pid = null;
try {
@@ -44,22 +45,25 @@ public class TestProcessIdFileReader {
assert(pid == null);
}
- @Test
+ @Test (timeout = 30000)
public void testSimpleGet() throws IOException {
String rootDir = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File testFile = null;
+ String expectedProcessId = Shell.WINDOWS ?
+ "container_1353742680940_0002_01_000001" :
+ "56789";
try {
testFile = new File(rootDir, "temp.txt");
PrintWriter fileWriter = new PrintWriter(testFile);
- fileWriter.println("56789");
+ fileWriter.println(expectedProcessId);
fileWriter.close();
String processId = null;
processId = ProcessIdFileReader.getProcessId(
new Path(rootDir + Path.SEPARATOR + "temp.txt"));
- Assert.assertEquals("56789", processId);
+ Assert.assertEquals(expectedProcessId, processId);
} finally {
if (testFile != null
@@ -70,12 +74,15 @@ public class TestProcessIdFileReader {
}
- @Test
+ @Test (timeout = 30000)
public void testComplexGet() throws IOException {
String rootDir = new File(System.getProperty(
"test.build.data", "/tmp")).getAbsolutePath();
File testFile = null;
-
+ String processIdInFile = Shell.WINDOWS ?
+ " container_1353742680940_0002_01_000001 " :
+ " 23 ";
+ String expectedProcessId = processIdInFile.trim();
try {
testFile = new File(rootDir, "temp.txt");
PrintWriter fileWriter = new PrintWriter(testFile);
@@ -84,14 +91,14 @@ public class TestProcessIdFileReader {
fileWriter.println("abc");
fileWriter.println("-123");
fileWriter.println("-123 ");
- fileWriter.println(" 23 ");
+ fileWriter.println(processIdInFile);
fileWriter.println("6236");
fileWriter.close();
String processId = null;
processId = ProcessIdFileReader.getProcessId(
new Path(rootDir + Path.SEPARATOR + "temp.txt"));
- Assert.assertEquals("23", processId);
+ Assert.assertEquals(expectedProcessId, processId);
} finally {
if (testFile != null