You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2012/11/08 20:09:59 UTC
svn commit: r1407217 [1/2] - in
/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project: ./
hadoop-yarn/conf/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-app...
Author: suresh
Date: Thu Nov 8 19:09:46 2012
New Revision: 1407217
URL: http://svn.apache.org/viewvc?rev=1407217&view=rev
Log:
Merging trunk to branch-trunk-win branch
Added:
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/
- copied from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
- copied unchanged from r1407201, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
Removed:
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.txt?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/CHANGES.txt Thu Nov 8 19:09:46 2012
@@ -23,6 +23,8 @@ Release 2.0.3-alpha - Unreleased
NEW FEATURES
+ YARN-145. Add a Web UI to the fair share scheduler. (Sandy Ryza via tomwhite)
+
IMPROVEMENTS
YARN-78. Changed UnManagedAM application to use YarnClient. (Bikas Saha via
@@ -86,7 +88,16 @@ Release 2.0.3-alpha - Unreleased
YARN-150. Fixes AppRejectedTransition does not unregister a rejected
app-attempt from the ApplicationMasterService (Bikas Saha via sseth)
- YARN-140. Add capacity-scheduler-default.xml to provide a default set of configurations for the capacity scheduler. (ahmed via tucu)
+ YARN-140. Add capacity-scheduler-default.xml to provide a default set of
+ configurations for the capacity scheduler. (ahmed via tucu)
+
+ YARN-179. Fix some unit test failures. (Vinod Kumar Vavilapalli via sseth)
+
+ YARN-181. Fixed eclipse settings broken by capacity-scheduler.xml move via
+ YARN-140. (Siddharth Seth via vinodkv)
+
+ YARN-169. Update log4j.appender.EventCounter to use
+ org.apache.hadoop.log.metrics.EventCounter (Anthony Rojas via tomwhite)
Release 2.0.2-alpha - 2012-09-07
@@ -158,6 +169,9 @@ Release 0.23.5 - UNRELEASED
YARN-32. Fix TestApplicationTokens to not depend on test order and thus pass
on JDK7. (vinodkv)
+ YARN-186. Coverage fixing LinuxContainerExecutor (Aleksey Gorshkov via
+ bobby)
+
OPTIMIZATIONS
BUG FIXES
@@ -165,6 +179,38 @@ Release 0.23.5 - UNRELEASED
YARN-163. Retrieving container log via NM webapp can hang with multibyte
characters in log (jlowe via bobby)
+ YARN-174. Modify NodeManager to pass the user's configuration even when
+ rebooting. (vinodkv)
+
+ YARN-177. CapacityScheduler - adding a queue while the RM is running has
+ wacky results (acmurthy vai tgraves)
+
+ YARN-178. Fix custom ProcessTree instance creation (Radim Kolar via bobby)
+
+ YARN-180. Capacity scheduler - containers that get reserved create
+ container token to early (acmurthy and bobby)
+
+ YARN-139. Interrupted Exception within AsyncDispatcher leads to user
+ confusion. (Vinod Kumar Vavilapalli via jlowe)
+
+ YARN-165. RM should point tracking URL to RM web page for app when AM fails
+ (jlowe via bobby)
+
+ YARN-159. RM web ui applications page should be sorted to display last app
+ first (tgraves via bobby)
+
+ YARN-166. capacity scheduler doesn't allow capacity < 1.0 (tgraves via
+ bobby)
+
+ YARN-189. Fixed a deadlock between RM's ApplicationMasterService and the
+ dispatcher. (Thomas Graves via vinodkv)
+
+ YARN-202. Log Aggregation generates a storm of fsync() for namenode
+ (Kihwal Lee via bobby)
+
+ YARN-201. Fix CapacityScheduler to be less conservative for starved
+ off-switch requests. (jlowe via acmurthy)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml Thu Nov 8 19:09:46 2012
@@ -53,6 +53,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<scope>test</scope>
</dependency>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java Thu Nov 8 19:09:46 2012
@@ -143,7 +143,7 @@ public class UnmanagedAMLauncher {
appName = cliParser.getOptionValue("appname", "UnmanagedAM");
amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
- amQueue = cliParser.getOptionValue("queue", "");
+ amQueue = cliParser.getOptionValue("queue", "default");
classpath = cliParser.getOptionValue("classpath", null);
amCmd = cliParser.getOptionValue("cmd");
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java Thu Nov 8 19:09:46 2012
@@ -141,6 +141,8 @@ public class TestUnmanagedAMLauncher {
String[] args = {
"--classpath",
classpath,
+ "--queue",
+ "default",
"--cmd",
javaHome
+ "/bin/java -Xmx512m "
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Thu Nov 8 19:09:46 2012
@@ -38,5 +38,12 @@
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
</dependencies>
</project>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Thu Nov 8 19:09:46 2012
@@ -68,7 +68,9 @@ public class AsyncDispatcher extends Abs
try {
event = eventQueue.take();
} catch(InterruptedException ie) {
- LOG.warn("AsyncDispatcher thread interrupted", ie);
+ if (!stopped) {
+ LOG.warn("AsyncDispatcher thread interrupted", ie);
+ }
return;
}
if (event != null) {
@@ -180,7 +182,9 @@ public class AsyncDispatcher extends Abs
try {
eventQueue.put(event);
} catch (InterruptedException e) {
- LOG.warn("AsyncDispatcher thread interrupted", e);
+ if (!stopped) {
+ LOG.warn("AsyncDispatcher thread interrupted", e);
+ }
throw new YarnException(e);
}
};
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java Thu Nov 8 19:09:46 2012
@@ -263,7 +263,6 @@ public class AggregatedLogFormat {
out = this.writer.prepareAppendValue(-1);
logValue.write(out);
out.close();
- this.fsDataOStream.hflush();
}
public void closeWriter() {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Thu Nov 8 19:09:46 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.util;
import java.util.Iterator;
import java.util.Map;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -109,6 +110,7 @@ public class Apps {
} else {
val = val + SYSTEM_PATH_SEPARATOR + value;
}
- environment.put(variable, val);
+ environment.put(StringInterner.weakIntern(variable),
+ StringInterner.weakIntern(val));
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Thu Nov 8 19:09:46 2012
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.RunJar;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -90,6 +91,85 @@ public class FSDownload implements Calla
}
}
+ /**
+ * Returns a boolean to denote whether a cache file is visible to all(public)
+ * or not
+ * @param conf
+ * @param uri
+ * @return true if the path in the uri is visible to all, false otherwise
+ * @throws IOException
+ */
+ private static boolean isPublic(FileSystem fs, Path current) throws IOException {
+ current = fs.makeQualified(current);
+ //the leaf level file should be readable by others
+ if (!checkPublicPermsForAll(fs, current, FsAction.READ_EXECUTE, FsAction.READ)) {
+ return false;
+ }
+ return ancestorsHaveExecutePermissions(fs, current.getParent());
+ }
+
+ private static boolean checkPublicPermsForAll(FileSystem fs, Path current,
+ FsAction dir, FsAction file)
+ throws IOException {
+ return checkPublicPermsForAll(fs, fs.getFileStatus(current), dir, file);
+ }
+
+ private static boolean checkPublicPermsForAll(FileSystem fs,
+ FileStatus status, FsAction dir, FsAction file)
+ throws IOException {
+ FsPermission perms = status.getPermission();
+ FsAction otherAction = perms.getOtherAction();
+ if (status.isDirectory()) {
+ if (!otherAction.implies(dir)) {
+ return false;
+ }
+
+ for (FileStatus child : fs.listStatus(status.getPath())) {
+ if(!checkPublicPermsForAll(fs, child, dir, file)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return (otherAction.implies(file));
+ }
+
+ /**
+ * Returns true if all ancestors of the specified path have the 'execute'
+ * permission set for all users (i.e. that other users can traverse
+ * the directory heirarchy to the given path)
+ */
+ private static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path)
+ throws IOException {
+ Path current = path;
+ while (current != null) {
+ //the subdirs in the path should have execute permissions for others
+ if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE)) {
+ return false;
+ }
+ current = current.getParent();
+ }
+ return true;
+ }
+
+ /**
+ * Checks for a given path whether the Other permissions on it
+ * imply the permission in the passed FsAction
+ * @param fs
+ * @param path
+ * @param action
+ * @return true if the path in the uri is visible to all, false otherwise
+ * @throws IOException
+ */
+ private static boolean checkPermissionOfOther(FileSystem fs, Path path,
+ FsAction action) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ FsPermission perms = status.getPermission();
+ FsAction otherAction = perms.getOtherAction();
+ return otherAction.implies(action);
+ }
+
+
private Path copy(Path sCopy, Path dstdir) throws IOException {
FileSystem sourceFs = sCopy.getFileSystem(conf);
Path dCopy = new Path(dstdir, sCopy.getName() + ".tmp");
@@ -99,7 +179,14 @@ public class FSDownload implements Calla
" changed on src filesystem (expected " + resource.getTimestamp() +
", was " + sStat.getModificationTime());
}
-
+ if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
+ if (!isPublic(sourceFs, sCopy)) {
+ throw new IOException("Resource " + sCopy +
+ " is not publicly accessable and as such cannot be part of the" +
+ " public cache.");
+ }
+ }
+
sourceFs.copyToLocalFile(sCopy, dCopy);
return dCopy;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Thu Nov 8 19:09:46 2012
@@ -114,6 +114,7 @@ public class ProcfsBasedProcessTree exte
* @param procfsDir the root of a proc file system - only used for testing.
*/
public ProcfsBasedProcessTree(String pid, String procfsDir) {
+ super(pid);
this.pid = getValidPID(pid);
this.procfsDir = procfsDir;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java Thu Nov 8 19:09:46 2012
@@ -21,17 +21,28 @@ package org.apache.hadoop.yarn.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.ReflectionUtils;
+import java.lang.reflect.Constructor;
/**
* Interface class to obtain process resource usage
*
*/
-public abstract class ResourceCalculatorProcessTree {
+public abstract class ResourceCalculatorProcessTree extends Configured {
static final Log LOG = LogFactory
.getLog(ResourceCalculatorProcessTree.class);
/**
+ * Create process-tree instance with specified root process.
+ *
+ * Subclass must override this.
+ * @param root process-tree root-process
+ */
+ public ResourceCalculatorProcessTree(String root) {
+ }
+
+ /**
* Get the process-tree with latest state. If the root-process is not alive,
* an empty tree will be returned.
*
@@ -122,10 +133,17 @@ public abstract class ResourceCalculator
* is not available for this system.
*/
public static ResourceCalculatorProcessTree getResourceCalculatorProcessTree(
- String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) {
+ String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) {
if (clazz != null) {
- return ReflectionUtils.newInstance(clazz, conf);
+ try {
+ Constructor <? extends ResourceCalculatorProcessTree> c = clazz.getConstructor(String.class);
+ ResourceCalculatorProcessTree rctree = c.newInstance(pid);
+ rctree.setConf(conf);
+ return rctree;
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ }
}
// No class given, try a os specific class
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java Thu Nov 8 19:09:46 2012
@@ -114,6 +114,54 @@ public class TestFSDownload {
}
@Test
+ public void testDownloadBadPublic() throws IOException, URISyntaxException,
+ InterruptedException {
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+ FileContext files = FileContext.getLocalFSFileContext(conf);
+ final Path basedir = files.makeQualified(new Path("target",
+ TestFSDownload.class.getSimpleName()));
+ files.mkdir(basedir, null, true);
+ conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
+
+ Map<LocalResource, LocalResourceVisibility> rsrcVis =
+ new HashMap<LocalResource, LocalResourceVisibility>();
+
+ Random rand = new Random();
+ long sharedSeed = rand.nextLong();
+ rand.setSeed(sharedSeed);
+ System.out.println("SEED: " + sharedSeed);
+
+ Map<LocalResource,Future<Path>> pending =
+ new HashMap<LocalResource,Future<Path>>();
+ ExecutorService exec = Executors.newSingleThreadExecutor();
+ LocalDirAllocator dirs =
+ new LocalDirAllocator(TestFSDownload.class.getName());
+ int size = 512;
+ LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
+ Path path = new Path(basedir, "test-file");
+ LocalResource rsrc = createFile(files, path, size, rand, vis);
+ rsrcVis.put(rsrc, vis);
+ Path destPath = dirs.getLocalPathForWrite(
+ basedir.toString(), size, conf);
+ FSDownload fsd =
+ new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
+ destPath, rsrc, new Random(sharedSeed));
+ pending.put(rsrc, exec.submit(fsd));
+
+ try {
+ for (Map.Entry<LocalResource,Future<Path>> p : pending.entrySet()) {
+ p.getValue().get();
+ Assert.fail("We localized a file that is not public.");
+ }
+ } catch (ExecutionException e) {
+ Assert.assertTrue(e.getCause() instanceof IOException);
+ } finally {
+ exec.shutdown();
+ }
+ }
+
+ @Test
public void testDownload() throws IOException, URISyntaxException,
InterruptedException {
Configuration conf = new Configuration();
@@ -140,14 +188,9 @@ public class TestFSDownload {
int[] sizes = new int[10];
for (int i = 0; i < 10; ++i) {
sizes[i] = rand.nextInt(512) + 512;
- LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
- switch (i%3) {
- case 1:
- vis = LocalResourceVisibility.PRIVATE;
- break;
- case 2:
+ LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
+ if (i%2 == 1) {
vis = LocalResourceVisibility.APPLICATION;
- break;
}
Path p = new Path(basedir, "" + i);
LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);
@@ -176,17 +219,8 @@ public class TestFSDownload {
System.out.println("File permission " + perm +
" for rsrc vis " + p.getKey().getVisibility().name());
assert(rsrcVis.containsKey(p.getKey()));
- switch (rsrcVis.get(p.getKey())) {
- case PUBLIC:
- Assert.assertTrue("Public file should be 555",
- perm.toShort() == FSDownload.PUBLIC_FILE_PERMS.toShort());
- break;
- case PRIVATE:
- case APPLICATION:
- Assert.assertTrue("Private file should be 500",
- perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
- break;
- }
+ Assert.assertTrue("Private file should be 500",
+ perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
}
} catch (ExecutionException e) {
throw new IOException("Failed exec", e);
@@ -250,14 +284,9 @@ public class TestFSDownload {
LocalDirAllocator dirs =
new LocalDirAllocator(TestFSDownload.class.getName());
for (int i = 0; i < 5; ++i) {
- LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
- switch (rand.nextInt()%3) {
- case 1:
- vis = LocalResourceVisibility.PRIVATE;
- break;
- case 2:
+ LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
+ if (i%2 == 1) {
vis = LocalResourceVisibility.APPLICATION;
- break;
}
Path p = new Path(basedir, "dir" + i + ".jar");
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Thu Nov 8 19:09:46 2012
@@ -253,12 +253,12 @@ public class NodeManager extends Composi
if (hasToReboot) {
LOG.info("Rebooting the node manager.");
NodeManager nodeManager = createNewNodeManager();
- nodeManager.initAndStartNodeManager(hasToReboot);
+ nodeManager.initAndStartNodeManager(this.getConfig(), hasToReboot);
}
}
}
- private void initAndStartNodeManager(boolean hasToReboot) {
+ private void initAndStartNodeManager(Configuration conf, boolean hasToReboot) {
try {
// Remove the old hook if we are rebooting.
@@ -270,7 +270,6 @@ public class NodeManager extends Composi
ShutdownHookManager.get().addShutdownHook(nodeManagerShutdownHook,
SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
this.init(conf);
this.start();
} catch (Throwable t) {
@@ -288,6 +287,7 @@ public class NodeManager extends Composi
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
NodeManager nodeManager = new NodeManager();
- nodeManager.initAndStartNodeManager(false);
+ Configuration conf = new YarnConfiguration();
+ nodeManager.initAndStartNodeManager(conf, false);
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties Thu Nov 8 19:09:46 2012
@@ -26,4 +26,4 @@ log4j.appender.CLA.layout.ConversionPatt
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
#
-log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Thu Nov 8 19:09:46 2012
@@ -26,11 +26,13 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
+import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -46,7 +48,6 @@ import org.junit.Test;
public class TestLinuxContainerExecutorWithMocks {
- @SuppressWarnings("unused")
private static final Log LOG = LogFactory
.getLog(TestLinuxContainerExecutorWithMocks.class);
@@ -54,6 +55,7 @@ public class TestLinuxContainerExecutorW
private final File mockParamFile = new File("./params.txt");
private LocalDirsHandlerService dirsHandler;
+
private void deleteMockParamFile() {
if(mockParamFile.exists()) {
mockParamFile.delete();
@@ -126,8 +128,102 @@ public class TestLinuxContainerExecutorW
StringUtils.join(",", dirsHandler.getLocalDirs()),
StringUtils.join(",", dirsHandler.getLogDirs())),
readMockParams());
+
+ }
+
+
+ @Test
+ public void testStartLocalizer() throws IOException {
+
+
+ InetSocketAddress address = InetSocketAddress.createUnresolved("localhost", 8040);
+ Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath");
+
+ try {
+ mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs());
+ List<String> result=readMockParams();
+ Assert.assertEquals(result.size(), 16);
+ Assert.assertEquals(result.get(0), "test");
+ Assert.assertEquals(result.get(1), "0" );
+ Assert.assertEquals(result.get(2),"application_0" );
+ Assert.assertEquals(result.get(3), "/bin/nmPrivateCTokensPath");
+ Assert.assertEquals(result.get(7), "-classpath" );
+ Assert.assertEquals(result.get(10),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
+ Assert.assertEquals(result.get(11), "test");
+ Assert.assertEquals(result.get(12), "application_0");
+ Assert.assertEquals(result.get(13),"12345" );
+ Assert.assertEquals(result.get(14),"localhost" );
+ Assert.assertEquals(result.get(15),"8040" );
+
+ } catch (InterruptedException e) {
+ LOG.error("Error:"+e.getMessage(),e);
+ Assert.fail();
+ }
+ }
+
+
+ @Test
+ public void testContainerLaunchError() throws IOException {
+
+ // reinitialize executer
+ File f = new File("./src/test/resources/mock-container-executer-with-error");
+ if (!f.canExecute()) {
+ f.setExecutable(true);
+ }
+ String executorPath = f.getAbsolutePath();
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
+ conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
+
+ mockExec = new LinuxContainerExecutor();
+ dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+ mockExec.setConf(conf);
+
+ String appSubmitter = "nobody";
+ String cmd = String
+ .valueOf(LinuxContainerExecutor.Commands.LAUNCH_CONTAINER.getValue());
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ Container container = mock(Container.class);
+ ContainerId cId = mock(ContainerId.class);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String, String>();
+
+ when(container.getContainerID()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+
+ when(cId.toString()).thenReturn(containerId);
+
+ when(context.getEnvironment()).thenReturn(env);
+
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+ Path workDir = new Path("/tmp");
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ mockExec.activateContainer(cId, pidFile);
+ int ret = mockExec.launchContainer(container, scriptPath, tokensPath,
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
+ Assert.assertNotSame(0, ret);
+ assertEquals(Arrays.asList(appSubmitter, cmd, appId, containerId,
+ workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
+ StringUtils.join(",", dirsHandler.getLocalDirs()),
+ StringUtils.join(",", dirsHandler.getLogDirs())), readMockParams());
+
}
+
+ @Test
+ public void testInit() throws Exception {
+ mockExec.init();
+ assertEquals(Arrays.asList("--checksetup"), readMockParams());
+
+ }
+
+
@Test
public void testContainerKill() throws IOException {
String appSubmitter = "nobody";
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Thu Nov 8 19:09:46 2012
@@ -50,7 +50,7 @@
<directory>${basedir}/src/test/resources</directory>
</testResource>
<testResource>
- <directory>${basedir}/../../conf</directory>
+ <directory>${basedir}/conf</directory>
<includes>
<include>capacity-scheduler.xml</include>
</includes>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Thu Nov 8 19:09:46 2012
@@ -265,10 +265,10 @@ public class ApplicationMasterService ex
// Oh damn! Sending reboot isn't enough. RM state is corrupted. TODO:
allocateResponse.setAMResponse(reboot);
return allocateResponse;
- }
-
+ }
+
// Allow only one thread in AM to do heartbeat at a time.
- synchronized (lastResponse) { // BUG TODO: Locking order is screwed.
+ synchronized (lastResponse) {
// Send the status update to the appAttempt.
this.rmContext.getDispatcher().getEventHandler().handle(
@@ -282,7 +282,8 @@ public class ApplicationMasterService ex
Allocation allocation =
this.rScheduler.allocate(appAttemptId, ask, release);
- RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId());
+ RMApp app = this.rmContext.getRMApps().get(
+ appAttemptId.getApplicationId());
RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
AMResponse response = recordFactory.newRecordInstance(AMResponse.class);
@@ -316,7 +317,18 @@ public class ApplicationMasterService ex
.pullJustFinishedContainers());
response.setResponseId(lastResponse.getResponseId() + 1);
response.setAvailableResources(allocation.getResourceLimit());
- responseMap.put(appAttemptId, response);
+
+ AMResponse oldResponse = responseMap.put(appAttemptId, response);
+ if (oldResponse == null) {
+ // appAttempt got unregistered, remove it back out
+ responseMap.remove(appAttemptId);
+ String message = "App Attempt removed from the cache during allocate"
+ + appAttemptId;
+ LOG.error(message);
+ allocateResponse.setAMResponse(reboot);
+ return allocateResponse;
+ }
+
allocateResponse.setAMResponse(response);
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
return allocateResponse;
@@ -331,12 +343,7 @@ public class ApplicationMasterService ex
}
public void unregisterAttempt(ApplicationAttemptId attemptId) {
- AMResponse lastResponse = responseMap.get(attemptId);
- if (lastResponse != null) {
- synchronized (lastResponse) {
- responseMap.remove(attemptId);
- }
- }
+ responseMap.remove(attemptId);
}
public void refreshServiceAcls(Configuration configuration,
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Thu Nov 8 19:09:46 2012
@@ -531,7 +531,7 @@ public class RMAppImpl implements RMApp
RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId,
clientTokenStr, rmContext, scheduler, masterService,
- submissionContext, YarnConfiguration.getProxyHostAndPort(conf));
+ submissionContext, conf);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
handler.handle(
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Thu Nov 8 19:09:46 2012
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
+import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
+
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -33,6 +35,7 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -45,6 +48,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -128,7 +132,7 @@ public class RMAppAttemptImpl implements
private FinalApplicationStatus finalStatus = null;
private final StringBuilder diagnostics = new StringBuilder();
- private final String proxy;
+ private Configuration conf;
private static final StateMachineFactory<RMAppAttemptImpl,
RMAppAttemptState,
@@ -285,9 +289,9 @@ public class RMAppAttemptImpl implements
String clientToken, RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- String proxy) {
+ Configuration conf) {
- this.proxy = proxy;
+ this.conf = conf;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
@@ -397,6 +401,7 @@ public class RMAppAttemptImpl implements
try {
URI trackingUri = trackingUriWithoutScheme == null ? null :
ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
+ String proxy = YarnConfiguration.getProxyHostAndPort(conf);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
applicationAttemptId.getApplicationId());
@@ -977,15 +982,13 @@ public class RMAppAttemptImpl implements
" due to: " + containerStatus.getDiagnostics() + "." +
"Failing this attempt.");
- /*
- * In the case when the AM dies, the trackingUrl is left pointing to the AM's
- * URL, which shows up in the scheduler UI as a broken link. Setting it here
- * to empty string will prevent any link from being displayed.
- * NOTE: don't set trackingUrl to 'null'. That will cause null-pointer exceptions
- * in the generated proto code.
- */
- appAttempt.origTrackingUrl = "";
- appAttempt.proxiedTrackingUrl = "";
+ // When the AM dies, the trackingUrl is left pointing to the AM's URL,
+ // which shows up in the scheduler UI as a broken link. Direct the
+ // user to the app page on the RM so they can see the status and logs.
+ appAttempt.origTrackingUrl = pjoin(
+ YarnConfiguration.getRMWebAppHostAndPort(appAttempt.conf),
+ "cluster", "app", appAttempt.getAppAttemptId().getApplicationId());
+ appAttempt.proxiedTrackingUrl = appAttempt.origTrackingUrl;
new FinalTransition(RMAppAttemptState.FAILED).transition(
appAttempt, containerFinishedEvent);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Thu Nov 8 19:09:46 2012
@@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re
public CSQueue getParent();
/**
+ * Set the parent <code>Queue</code>.
+ * @param newParentQueue new parent queue
+ */
+ public void setParent(CSQueue newParentQueue);
+
+ /**
* Get the queue name.
* @return the queue name
*/
@@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re
/**
* Reinitialize the queue.
- * @param queue new queue to re-initalize from
+ * @param newlyParsedQueue new queue to re-initalize from
* @param clusterResource resources in the cluster
*/
- public void reinitialize(CSQueue queue, Resource clusterResource)
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException;
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Thu Nov 8 19:09:46 2012
@@ -84,7 +84,7 @@ class CSQueueUtils {
if (clusterMemory > 0) {
queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
absoluteUsedCapacity = ((float)usedMemory / (float)clusterMemory);
- usedCapacity = (usedMemory / queueLimit);
+ usedCapacity = (queueLimit == 0) ? 0 : (usedMemory / queueLimit);
}
childQueue.setUsedCapacity(usedCapacity);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Thu Nov 8 19:09:46 2012
@@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -111,21 +110,18 @@ implements ResourceScheduler, CapacitySc
}
};
+ @Override
public void setConf(Configuration conf) {
- if (conf instanceof YarnConfiguration) {
- yarnConf = (YarnConfiguration) conf;
- } else {
- throw new IllegalArgumentException("Can only configure with " +
- "YarnConfiguration");
- }
+ yarnConf = conf;
}
+ @Override
public Configuration getConf() {
return yarnConf;
}
private CapacitySchedulerConfiguration conf;
- private YarnConfiguration yarnConf;
+ private Configuration yarnConf;
private RMContext rmContext;
private Map<String, CSQueue> queues = new ConcurrentHashMap<String, CSQueue>();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Thu Nov 8 19:09:46 2012
@@ -91,7 +91,7 @@ public class CapacitySchedulerConfigurat
public static final float UNDEFINED = -1;
@Private
- public static final float MINIMUM_CAPACITY_VALUE = 1;
+ public static final float MINIMUM_CAPACITY_VALUE = 0;
@Private
public static final float MAXIMUM_CAPACITY_VALUE = 100;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Nov 8 19:09:46 2012
@@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- float absCapacity = parent.getAbsoluteCapacity() * capacity;
+ float absCapacity = getParent().getAbsoluteCapacity() * capacity;
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absCapacity, absoluteMaxCapacity);
this.capacity = capacity;
@@ -256,7 +256,7 @@ public class LeafQueue implements CSQueu
// Update metrics
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
LOG.info("Initializing " + queueName + "\n" +
"capacity = " + capacity +
@@ -339,10 +339,15 @@ public class LeafQueue implements CSQueu
}
@Override
- public CSQueue getParent() {
+ public synchronized CSQueue getParent() {
return parent;
}
-
+
+ @Override
+ public synchronized void setParent(CSQueue newParentQueue) {
+ this.parent = (ParentQueue)newParentQueue;
+ }
+
@Override
public String getQueueName() {
return queueName;
@@ -350,7 +355,7 @@ public class LeafQueue implements CSQueu
@Override
public String getQueuePath() {
- return parent.getQueuePath() + "." + getQueueName();
+ return getParent().getQueuePath() + "." + getQueueName();
}
/**
@@ -430,7 +435,9 @@ public class LeafQueue implements CSQueu
synchronized void setMaxCapacity(float maximumCapacity) {
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
+ float absMaxCapacity =
+ CSQueueUtils.computeAbsoluteMaximumCapacity(
+ maximumCapacity, getParent());
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absMaxCapacity);
this.maximumCapacity = maximumCapacity;
@@ -453,10 +460,6 @@ public class LeafQueue implements CSQueu
this.userLimitFactor = userLimitFactor;
}
- synchronized void setParentQueue(CSQueue parent) {
- this.parent = parent;
- }
-
@Override
public synchronized int getNumApplications() {
return getNumPendingApplications() + getNumActiveApplications();
@@ -559,26 +562,28 @@ public class LeafQueue implements CSQueu
}
@Override
- public synchronized void reinitialize(CSQueue queue, Resource clusterResource)
+ public synchronized void reinitialize(
+ CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
// Sanity check
- if (!(queue instanceof LeafQueue) ||
- !queue.getQueuePath().equals(getQueuePath())) {
+ if (!(newlyParsedQueue instanceof LeafQueue) ||
+ !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
throw new IOException("Trying to reinitialize " + getQueuePath() +
- " from " + queue.getQueuePath());
+ " from " + newlyParsedQueue.getQueuePath());
}
- LeafQueue leafQueue = (LeafQueue)queue;
+ LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;
setupQueueConfigs(
clusterResource,
- leafQueue.capacity, leafQueue.absoluteCapacity,
- leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity,
- leafQueue.userLimit, leafQueue.userLimitFactor,
- leafQueue.maxApplications,
- leafQueue.getMaxApplicationsPerUser(),
- leafQueue.getMaximumActiveApplications(),
- leafQueue.getMaximumActiveApplicationsPerUser(),
- leafQueue.state, leafQueue.acls);
+ newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity,
+ newlyParsedLeafQueue.maximumCapacity,
+ newlyParsedLeafQueue.absoluteMaxCapacity,
+ newlyParsedLeafQueue.userLimit, newlyParsedLeafQueue.userLimitFactor,
+ newlyParsedLeafQueue.maxApplications,
+ newlyParsedLeafQueue.getMaxApplicationsPerUser(),
+ newlyParsedLeafQueue.getMaximumActiveApplications(),
+ newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
+ newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls);
}
@Override
@@ -591,7 +596,7 @@ public class LeafQueue implements CSQueu
}
// Check if parent-queue allows access
- return parent.hasAccess(acl, user);
+ return getParent().hasAccess(acl, user);
}
@Override
@@ -649,10 +654,10 @@ public class LeafQueue implements CSQueu
// Inform the parent queue
try {
- parent.submitApplication(application, userName, queue);
+ getParent().submitApplication(application, userName, queue);
} catch (AccessControlException ace) {
LOG.info("Failed to submit application to parent-queue: " +
- parent.getQueuePath(), ace);
+ getParent().getQueuePath(), ace);
removeApplication(application, user);
throw ace;
}
@@ -708,7 +713,7 @@ public class LeafQueue implements CSQueu
}
// Inform the parent queue
- parent.finishApplication(application, queue);
+ getParent().finishApplication(application, queue);
}
public synchronized void removeApplication(FiCaSchedulerApp application, User user) {
@@ -828,8 +833,12 @@ public class LeafQueue implements CSQueu
// Note: Update headroom to account for current allocation too...
allocateResource(clusterResource, application, assigned);
- // Reset scheduling opportunities
- application.resetSchedulingOpportunities(priority);
+ // Don't reset scheduling opportunities for non-local assignments
+ // otherwise the app will be delayed for each non-local assignment.
+ // This helps apps with many off-cluster requests schedule faster.
+ if (assignment.getType() != NodeType.OFF_SWITCH) {
+ application.resetSchedulingOpportunities(priority);
+ }
// Done
return assignment;
@@ -1183,34 +1192,32 @@ public class LeafQueue implements CSQueu
return (rmContainer != null) ? rmContainer.getContainer() :
createContainer(application, node, capability, priority);
}
-
- public Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node,
+ Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node,
Resource capability, Priority priority) {
-
+
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(application
.getApplicationAttemptId(), application.getNewContainerId());
- ContainerToken containerToken = null;
-
- // If security is enabled, send the container-tokens too.
- if (UserGroupInformation.isSecurityEnabled()) {
- containerToken =
- containerTokenSecretManager.createContainerToken(containerId, nodeId,
- application.getUser(), capability);
- if (containerToken == null) {
- return null; // Try again later.
- }
- }
-
+
// Create the container
Container container = BuilderUtils.newContainer(containerId, nodeId,
node.getRMNode().getHttpAddress(), capability, priority,
- containerToken);
-
+ null);
+
return container;
}
-
+
+ /**
+ * Create <code>ContainerToken</code>, only in secure-mode
+ */
+ ContainerToken createContainerToken(
+ FiCaSchedulerApp application, Container container) {
+ return containerTokenSecretManager.createContainerToken(
+ container.getId(), container.getNodeId(),
+ application.getUser(), container.getResource());
+ }
+
private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode node,
FiCaSchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
@@ -1246,6 +1253,17 @@ public class LeafQueue implements CSQueu
unreserve(application, priority, node, rmContainer);
}
+ // Create container tokens in secure-mode
+ if (UserGroupInformation.isSecurityEnabled()) {
+ ContainerToken containerToken =
+ createContainerToken(application, container);
+ if (containerToken == null) {
+ // Something went wrong...
+ return Resources.none();
+ }
+ container.setContainerToken(containerToken);
+ }
+
// Inform the application
RMContainer allocatedContainer =
application.allocate(type, node, priority, request, container);
@@ -1351,7 +1369,7 @@ public class LeafQueue implements CSQueu
}
// Inform the parent queue
- parent.completedContainer(clusterResource, application,
+ getParent().completedContainer(clusterResource, application,
node, rmContainer, null, event);
}
}
@@ -1361,7 +1379,7 @@ public class LeafQueue implements CSQueu
// Update queue metrics
Resources.addTo(usedResources, resource);
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
++numContainers;
// Update user metrics
@@ -1386,7 +1404,7 @@ public class LeafQueue implements CSQueu
// Update queue metrics
Resources.subtractFrom(usedResources, resource);
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
--numContainers;
// Update user metrics
@@ -1417,7 +1435,7 @@ public class LeafQueue implements CSQueu
// Update metrics
CSQueueUtils.updateQueueStatistics(
- this, parent, clusterResource, minimumAllocation);
+ this, getParent(), clusterResource, minimumAllocation);
// Update application properties
for (FiCaSchedulerApp application : activeApplications) {
@@ -1488,7 +1506,7 @@ public class LeafQueue implements CSQueu
synchronized (this) {
allocateResource(clusterResource, application, container.getResource());
}
- parent.recoverContainer(clusterResource, application, container);
+ getParent().recoverContainer(clusterResource, application, container);
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Thu Nov 8 19:09:46 2012
@@ -60,7 +60,7 @@ public class ParentQueue implements CSQu
private static final Log LOG = LogFactory.getLog(ParentQueue.class);
- private final CSQueue parent;
+ private CSQueue parent;
private final String queueName;
private float capacity;
@@ -202,7 +202,9 @@ public class ParentQueue implements CSQu
childCapacities += queue.getCapacity();
}
float delta = Math.abs(1.0f - childCapacities); // crude way to check
- if (delta > PRECISION) {
+ // allow capacities being set to 0, and enforce child 0 if parent is 0
+ if (((capacity > 0) && (delta > PRECISION)) ||
+ ((capacity == 0) && (childCapacities > 0))) {
throw new IllegalArgumentException("Illegal" +
" capacity of " + childCapacities +
" for children of queue " + queueName);
@@ -216,11 +218,16 @@ public class ParentQueue implements CSQu
}
@Override
- public CSQueue getParent() {
+ public synchronized CSQueue getParent() {
return parent;
}
@Override
+ public synchronized void setParent(CSQueue newParentQueue) {
+ this.parent = (ParentQueue)newParentQueue;
+ }
+
+ @Override
public String getQueueName() {
return queueName;
}
@@ -357,37 +364,52 @@ public class ParentQueue implements CSQu
}
@Override
- public synchronized void reinitialize(CSQueue queue, Resource clusterResource)
+ public synchronized void reinitialize(
+ CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
// Sanity check
- if (!(queue instanceof ParentQueue) ||
- !queue.getQueuePath().equals(getQueuePath())) {
+ if (!(newlyParsedQueue instanceof ParentQueue) ||
+ !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
throw new IOException("Trying to reinitialize " + getQueuePath() +
- " from " + queue.getQueuePath());
+ " from " + newlyParsedQueue.getQueuePath());
}
- ParentQueue parentQueue = (ParentQueue)queue;
+ ParentQueue newlyParsedParentQueue = (ParentQueue)newlyParsedQueue;
// Set new configs
setupQueueConfigs(clusterResource,
- parentQueue.capacity, parentQueue.absoluteCapacity,
- parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity,
- parentQueue.state, parentQueue.acls);
+ newlyParsedParentQueue.capacity,
+ newlyParsedParentQueue.absoluteCapacity,
+ newlyParsedParentQueue.maximumCapacity,
+ newlyParsedParentQueue.absoluteMaxCapacity,
+ newlyParsedParentQueue.state,
+ newlyParsedParentQueue.acls);
// Re-configure existing child queues and add new ones
// The CS has already checked to ensure all existing child queues are present!
Map<String, CSQueue> currentChildQueues = getQueues(childQueues);
- Map<String, CSQueue> newChildQueues = getQueues(parentQueue.childQueues);
+ Map<String, CSQueue> newChildQueues =
+ getQueues(newlyParsedParentQueue.childQueues);
for (Map.Entry<String, CSQueue> e : newChildQueues.entrySet()) {
String newChildQueueName = e.getKey();
CSQueue newChildQueue = e.getValue();
CSQueue childQueue = currentChildQueues.get(newChildQueueName);
- if (childQueue != null){
+
+ // Check if the child-queue already exists
+ if (childQueue != null) {
+ // Re-init existing child queues
childQueue.reinitialize(newChildQueue, clusterResource);
LOG.info(getQueueName() + ": re-configured queue: " + childQueue);
} else {
+ // New child queue, do not re-init
+
+ // Set parent to 'this'
+ newChildQueue.setParent(this);
+
+ // Save in list of current child queues
currentChildQueues.put(newChildQueueName, newChildQueue);
+
LOG.info(getQueueName() + ": added new child queue: " + newChildQueue);
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Thu Nov 8 19:09:46 2012
@@ -56,7 +56,9 @@ public class FSQueue {
public void addApp(FSSchedulerApp app) {
applications.add(app);
- queueSchedulable.addApp(new AppSchedulable(scheduler, app, this));
+ AppSchedulable appSchedulable = new AppSchedulable(scheduler, app, this);
+ app.setAppSchedulable(appSchedulable);
+ queueSchedulable.addApp(appSchedulable);
}
public void removeJob(FSSchedulerApp app) {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java Thu Nov 8 19:09:46 2012
@@ -70,6 +70,7 @@ public class FSSchedulerApp extends Sche
.getRecordFactory(null);
private final AppSchedulingInfo appSchedulingInfo;
+ private AppSchedulable appSchedulable;
private final Queue queue;
private final Resource currentConsumption = recordFactory
@@ -118,6 +119,14 @@ public class FSSchedulerApp extends Sche
public ApplicationAttemptId getApplicationAttemptId() {
return this.appSchedulingInfo.getApplicationAttemptId();
}
+
+ public void setAppSchedulable(AppSchedulable appSchedulable) {
+ this.appSchedulable = appSchedulable;
+ }
+
+ public AppSchedulable getAppSchedulable() {
+ return appSchedulable;
+ }
public String getUser() {
return this.appSchedulingInfo.getUser();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu Nov 8 19:09:46 2012
@@ -788,7 +788,11 @@ public class FairScheduler implements Re
FSSchedulerNode node = nodes.get(nodeId);
return node == null ? null : new SchedulerNodeReport(node);
}
-
+
+ public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) {
+ return applications.get(appAttemptId);
+ }
+
@Override
public SchedulerAppReport getSchedulerAppInfo(
ApplicationAttemptId appAttemptId) {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Thu Nov 8 19:09:46 2012
@@ -446,7 +446,7 @@ public class QueueManager {
* Get the maximum resource allocation for the given queue.
* @return the cap set on this queue, or Integer.MAX_VALUE if not set.
*/
- Resource getMaxResources(String queueName) {
+ public Resource getMaxResources(String queueName) {
synchronized (maxQueueResourcesMO) {
if (maxQueueResources.containsKey(queueName)) {
return maxQueueResources.get(queueName);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java Thu Nov 8 19:09:46 2012
@@ -77,8 +77,9 @@ public class RmController extends Contro
}
if (rs instanceof FairScheduler) {
- context().setStatus(404);
- throw new WebAppException("Fair Scheduler UI not yet supported");
+ setTitle("Fair Scheduler");
+ render(FairSchedulerPage.class);
+ return;
}
setTitle("Default Scheduler");