You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/04/18 18:32:56 UTC
svn commit: r1588509 [1/2] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop...
Author: cmccabe
Date: Fri Apr 18 16:32:35 2014
New Revision: 1588509
URL: http://svn.apache.org/r1588509
Log:
Merge trunk into the HADOOP-10388 branch
Added:
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
- copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm
- copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRestart.apt.vm
- copied unchanged from r1588387, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRestart.apt.vm
Modified:
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YARN.apt.vm
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/CHANGES.txt Fri Apr 18 16:32:35 2014
@@ -21,6 +21,8 @@ Release 2.5.0 - UNRELEASED
NEW FEATURES
+ YARN-1757. NM Recovery. Auxiliary service support. (Jason Lowe via kasha)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -36,9 +38,6 @@ Release 2.5.0 - UNRELEASED
YARN-1429. *nix: Allow a way for users to augment classpath of YARN daemons.
(Jarek Jarcec Cecho via kasha)
- YARN-1833. TestRMAdminService Fails in trunk and branch-2 (Mit Desais via
- jeagles)
-
YARN-1520. update capacity scheduler docs to include necessary parameters
(Chen He via jeagles)
@@ -48,6 +47,15 @@ Release 2.5.0 - UNRELEASED
YARN-1136. Replace junit.framework.Assert with org.junit.Assert (Chen He
via jeagles)
+ YARN-1889. In Fair Scheduler, avoid creating objects on each call to
+ AppSchedulable comparator (Hong Zhiguo via Sandy Ryza)
+
+ YARN-1923. Make Fair Scheduler resource ratio calculations terminate faster
+ (Anubhav Dhoot via Sandy Ryza)
+
+ YARN-1870. FileInputStream is not closed in ProcfsBasedProcessTree#constructProcessSMAPInfo.
+ (Fengdong Yu via junping_du)
+
OPTIMIZATIONS
BUG FIXES
@@ -57,7 +65,86 @@ Release 2.5.0 - UNRELEASED
YARN-1790. Fair Scheduler UI not showing apps table (bc Wong via Sandy Ryza)
-Release 2.4.0 - UNRELEASED
+ YARN-1784. TestContainerAllocation assumes CapacityScheduler.
+ (Robert Kanter via kasha)
+
+Release 2.4.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ YARN-1898. Made Standby RM links conf, stacks, logLevel, metrics, jmx, logs
+ and static not be redirected to Active RM. (Xuan Gong via zjshen)
+
+ YARN-1837. Fixed TestMoveApplication#testMoveRejectedByScheduler failure.
+ (Hong Zhiguo via jianhe)
+
+ YARN-1905. TestProcfsBasedProcessTree must only run on Linux. (cnauroth)
+
+ YARN-1883. TestRMAdminService fails due to inconsistent entries in
+ UserGroups (Mit Desai via jeagles)
+
+ YARN-1908. Fixed DistributedShell to not fail in secure clusters. (Vinod
+ Kumar Vavilapalli and Jian He via vinodkv)
+
+ YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to
+ fail more often on Windows. (Xuan Gong via vinodkv)
+
+ YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows.
+ (Vinod Kumar Vavilapalli via zjshen)
+
+ YARN-1914. Fixed resource-download on NodeManagers to skip permission
+ verification of public cache files in Windows+local file-system environment.
+ (Varun Vasudev via vinodkv)
+
+ YARN-1903. Set exit code and diagnostics when container is killed at
+ NEW/LOCALIZING state. (Zhijie Shen via jianhe)
+
+ YARN-1924. Made ZKRMStateStore updateApplication(Attempt)StateInternal work
+ when Application(Attempt) state hasn't been stored before. (Jian He via
+ zjshen)
+
+ YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for
+ HDFS paths and thus avoid test failures on Windows. (Varun Vasudev via
+ vinodkv)
+
+ YARN-1833. TestRMAdminService Fails in trunk and branch-2 (Mit Desai via
+ jeagles)
+
+ YARN-1907. TestRMApplicationHistoryWriter#testRMWritingMassiveHistory
+ intermittently fails. (Mit Desai via kihwal)
+
+ YARN-1933. Fixed test issues with TestAMRestart and TestNodeHealthService.
+ (Jian He via vinodkv)
+
+ YARN-1928. Fixed a race condition in TestAMRMRPCNodeUpdates which caused it
+ to fail occassionally. (Zhijie Shen via vinodkv)
+
+ YARN-1934. Fixed a potential NPE in ZKRMStateStore caused by handling
+ Disconnected event from ZK. (Karthik Kambatla via jianhe)
+
+ YARN-1931. Private API change in YARN-1824 in 2.4 broke compatibility
+ with previous releases (Sandy Ryza via tgraves)
+
+ YARN-1750. TestNodeStatusUpdater#testNMRegistration is incorrect in test
+ case. (Wangda Tan via junping_du)
+
+ YARN-1947. TestRMDelegationTokens#testRMDTMasterKeyStateOnRollingMasterKey
+ is failing intermittently. (Jian He via junping_du)
+
+ YARN-1281. Fixed TestZKRMStateStoreZKClientConnections to not fail
+ intermittently due to ZK-client timeouts. (Tsuyoshi Ozawa via vinodkv)
+
+Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
@@ -329,6 +416,11 @@ Release 2.4.0 - UNRELEASED
YARN-1452. Added documentation about the configuration and usage of generic
application history and the timeline data service. (Zhijie Shen via vinodkv)
+ YARN-1891. Added documentation for NodeManager health-monitoring. (Varun
+ Vasudev via vinodkv)
+
+ YARN-1017. Added documentation for ResourceManager Restart.(jianhe)
+
OPTIMIZATIONS
YARN-1771. Reduce the number of NameNode operations during localization of
@@ -588,6 +680,9 @@ Release 2.4.0 - UNRELEASED
YARN-1873. Fixed TestDistributedShell failure when the test cases are out of
order. (Mit Desai via zjshen)
+ YARN-1893. Mark AtMostOnce annotation to ApplicationMasterProtocol#allocate.
+ (Xuan Gong via jianhe)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java Fri Apr 18 16:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -162,6 +163,7 @@ public interface ApplicationMasterProtoc
*/
@Public
@Stable
+ @AtMostOnce
public AllocateResponse allocate(AllocateRequest request)
throws YarnException, IOException;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Apr 18 16:32:35 2014
@@ -884,6 +884,13 @@ public class YarnConfiguration extends C
public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
+ public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery.";
+ public static final String NM_RECOVERY_ENABLED =
+ NM_RECOVERY_PREFIX + "enabled";
+ public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false;
+
+ public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir";
+
////////////////////////////////
// Web Proxy Configs
////////////////////////////////
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java Fri Apr 18 16:32:35 2014
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -38,11 +39,22 @@ import org.apache.hadoop.yarn.conf.YarnC
@Evolving
public abstract class AuxiliaryService extends AbstractService {
+ private Path recoveryPath = null;
+
protected AuxiliaryService(String name) {
super(name);
}
/**
+ * Get the path specific to this auxiliary service to use for recovery.
+ *
+ * @return state storage path or null if recovery is not enabled
+ */
+ protected Path getRecoveryPath() {
+ return recoveryPath;
+ }
+
+ /**
* A new application is started on this NodeManager. This is a signal to
* this {@link AuxiliaryService} about the application initialization.
*
@@ -102,4 +114,13 @@ public abstract class AuxiliaryService e
public void stopContainer(ContainerTerminationContext stopContainerContext) {
}
+ /**
+ * Set the path for this auxiliary service to use for storing state
+ * that will be used during recovery.
+ *
+ * @param recoveryPath where recoverable state should be stored
+ */
+ public void setRecoveryPath(Path recoveryPath) {
+ this.recoveryPath = recoveryPath;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Apr 18 16:32:35 2014
@@ -27,6 +27,7 @@ import java.io.StringReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -91,7 +92,6 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -185,6 +185,9 @@ public class ApplicationMaster {
@SuppressWarnings("rawtypes")
private AMRMClientAsync amRMClient;
+ // In both secure and non-secure modes, this points to the job-submitter.
+ private UserGroupInformation appSubmitterUgi;
+
// Handle to communicate with the Node Manager
private NMClientAsync nmClientAsync;
// Listen to process the response from the Node Manager
@@ -236,7 +239,7 @@ public class ApplicationMaster {
// Location of shell script ( obtained from info set in env )
// Shell script path in fs
- private String shellScriptPath = "";
+ private String scriptPath = "";
// Timestamp needed for creating a local resource
private long shellScriptPathTimestamp = 0;
// File length needed for local resource
@@ -451,7 +454,7 @@ public class ApplicationMaster {
}
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) {
- shellScriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
+ scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) {
shellScriptPathTimestamp = Long.valueOf(envs
@@ -462,10 +465,10 @@ public class ApplicationMaster {
.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
- if (!shellScriptPath.isEmpty()
+ if (!scriptPath.isEmpty()
&& (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) {
LOG.error("Illegal values in env for shell script path" + ", path="
- + shellScriptPath + ", len=" + shellScriptPathLen + ", timestamp="
+ + scriptPath + ", len=" + shellScriptPathLen + ", timestamp="
+ shellScriptPathTimestamp);
throw new IllegalArgumentException(
"Illegal values in env for shell script path");
@@ -525,14 +528,23 @@ public class ApplicationMaster {
credentials.writeTokenStorageToStream(dob);
// Now remove the AM->RM token so that containers cannot access it.
Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ LOG.info("Executing with tokens:");
while (iter.hasNext()) {
Token<?> token = iter.next();
+ LOG.info(token);
if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
iter.remove();
}
}
allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ // Create appSubmitterUgi and add original tokens to it
+ String appSubmitterUserName =
+ System.getenv(ApplicationConstants.Environment.USER.name());
+ appSubmitterUgi =
+ UserGroupInformation.createRemoteUser(appSubmitterUserName);
+ appSubmitterUgi.addCredentials(credentials);
+
AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
amRMClient.init(conf);
@@ -901,19 +913,26 @@ public class ApplicationMaster {
// resources too.
// In this scenario, if a shell script is specified, we need to have it
// copied and made available to the container.
- if (!shellScriptPath.isEmpty()) {
- Path renamedSchellScriptPath = null;
+ if (!scriptPath.isEmpty()) {
+ Path renamedScriptPath = null;
if (Shell.WINDOWS) {
- renamedSchellScriptPath = new Path(shellScriptPath + ".bat");
+ renamedScriptPath = new Path(scriptPath + ".bat");
} else {
- renamedSchellScriptPath = new Path(shellScriptPath + ".sh");
+ renamedScriptPath = new Path(scriptPath + ".sh");
}
+
try {
- FileSystem fs = renamedSchellScriptPath.getFileSystem(conf);
- fs.rename(new Path(shellScriptPath), renamedSchellScriptPath);
- } catch (IOException e) {
- LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename");
- throw new YarnRuntimeException(e);
+ // rename the script file based on the underlying OS syntax.
+ renameScriptFile(renamedScriptPath);
+ } catch (Exception e) {
+ LOG.error(
+ "Not able to add suffix (.bat/.sh) to the shell script filename",
+ e);
+ // We know we cannot continue launching the container
+ // so we should release it.
+ numCompletedContainers.incrementAndGet();
+ numFailedContainers.incrementAndGet();
+ return;
}
LocalResource shellRsrc = Records.newRecord(LocalResource.class);
@@ -921,11 +940,10 @@ public class ApplicationMaster {
shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
try {
shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(
- renamedSchellScriptPath.toString())));
+ renamedScriptPath.toString())));
} catch (URISyntaxException e) {
LOG.error("Error when trying to use shell script path specified"
- + " in env, path=" + renamedSchellScriptPath);
- e.printStackTrace();
+ + " in env, path=" + renamedScriptPath, e);
// A failure scenario on bad input such as invalid shell script path
// We know we cannot continue launching the container
@@ -949,7 +967,7 @@ public class ApplicationMaster {
// Set executable command
vargs.add(shellCommand);
// Set shell script path
- if (!shellScriptPath.isEmpty()) {
+ if (!scriptPath.isEmpty()) {
vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath
: ExecShellStringPath);
}
@@ -983,6 +1001,20 @@ public class ApplicationMaster {
}
}
+ private void renameScriptFile(final Path renamedScriptPath)
+ throws IOException, InterruptedException {
+ appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException {
+ FileSystem fs = renamedScriptPath.getFileSystem(conf);
+ fs.rename(new Path(scriptPath), renamedScriptPath);
+ return null;
+ }
+ });
+ LOG.info("User " + appSubmitterUgi.getUserName()
+ + " added suffix(.sh/.bat) to script file as " + renamedScriptPath);
+ }
+
/**
* Setup the request that will be sent to the RM for the container ask.
*
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Fri Apr 18 16:32:35 2014
@@ -468,12 +468,12 @@ public class Client {
// Copy the application master jar to the filesystem
// Create a local resource to point to the destination jar path
FileSystem fs = FileSystem.get(conf);
- addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(),
+ addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(),
localResources, null);
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
- addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(),
+ addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(),
localResources, null);
}
@@ -489,7 +489,7 @@ public class Client {
if (!shellScriptPath.isEmpty()) {
Path shellSrc = new Path(shellScriptPath);
String shellPathSuffix =
- appName + "/" + appId.getId() + "/" + SCRIPT_PATH;
+ appName + "/" + appId.toString() + "/" + SCRIPT_PATH;
Path shellDst =
new Path(fs.getHomeDirectory(), shellPathSuffix);
fs.copyFromLocalFile(false, true, shellSrc, shellDst);
@@ -500,12 +500,12 @@ public class Client {
}
if (!shellCommand.isEmpty()) {
- addToLocalResources(fs, null, shellCommandPath, appId.getId(),
+ addToLocalResources(fs, null, shellCommandPath, appId.toString(),
localResources, shellCommand);
}
if (shellArgs.length > 0) {
- addToLocalResources(fs, null, shellArgsPath, appId.getId(),
+ addToLocalResources(fs, null, shellArgsPath, appId.toString(),
localResources, StringUtils.join(shellArgs, " "));
}
// Set local resource info into app master container launch context
@@ -737,7 +737,7 @@ public class Client {
}
private void addToLocalResources(FileSystem fs, String fileSrcPath,
- String fileDstPath, int appId, Map<String, LocalResource> localResources,
+ String fileDstPath, String appId, Map<String, LocalResource> localResources,
String resources) throws IOException {
String suffix =
appName + "/" + appId + "/" + fileDstPath;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Fri Apr 18 16:32:35 2014
@@ -219,7 +219,7 @@ public class TestDistributedShell {
"--num_containers",
"1",
"--shell_command",
- Shell.WINDOWS ? "timeout 8" : "sleep 8",
+ "sleep 8",
"--master_memory",
"512",
"--container_memory",
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java Fri Apr 18 16:32:35 2014
@@ -33,6 +33,8 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@@ -67,14 +69,18 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -82,6 +88,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -96,6 +103,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager;
@@ -257,11 +265,13 @@ public abstract class ProtocolHATestBase
}
protected void startHACluster(int numOfNMs, boolean overrideClientRMService,
- boolean overrideRTS) throws Exception {
+ boolean overrideRTS, boolean overrideApplicationMasterService)
+ throws Exception {
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
- numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS);
+ numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
+ overrideApplicationMasterService);
cluster.resetStartFailoverFlag(false);
cluster.init(conf);
cluster.start();
@@ -285,17 +295,19 @@ public abstract class ProtocolHATestBase
private boolean overrideClientRMService;
private boolean overrideRTS;
+ private boolean overrideApplicationMasterService;
private final AtomicBoolean startFailover = new AtomicBoolean(false);
private final AtomicBoolean failoverTriggered = new AtomicBoolean(false);
public MiniYARNClusterForHATesting(String testName,
int numResourceManagers, int numNodeManagers, int numLocalDirs,
int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
- boolean overrideRTS) {
+ boolean overrideRTS, boolean overrideApplicationMasterService) {
super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
numLogDirs, enableAHS);
this.overrideClientRMService = overrideClientRMService;
this.overrideRTS = overrideRTS;
+ this.overrideApplicationMasterService = overrideApplicationMasterService;
}
public boolean getStartFailoverFlag() {
@@ -324,6 +336,11 @@ public abstract class ProtocolHATestBase
if (count >= maximumWaittingTime) {
return false;
}
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ // DO NOTHING
+ }
return true;
}
@@ -354,6 +371,14 @@ public abstract class ProtocolHATestBase
}
return super.createResourceTrackerService();
}
+ @Override
+ protected ApplicationMasterService createApplicationMasterService() {
+ if (overrideApplicationMasterService) {
+ return new CustomedApplicationMasterService(this.rmContext,
+ this.scheduler);
+ }
+ return super.createApplicationMasterService();
+ }
};
}
@@ -717,5 +742,31 @@ public abstract class ProtocolHATestBase
return super.nodeHeartbeat(request);
}
}
+
+ private class CustomedApplicationMasterService extends
+ ApplicationMasterService {
+ public CustomedApplicationMasterService(RMContext rmContext,
+ YarnScheduler scheduler) {
+ super(rmContext, scheduler);
+ }
+
+ @Override
+ public AllocateResponse allocate(AllocateRequest request)
+ throws YarnException, IOException {
+ resetStartFailoverFlag(true);
+ // make sure failover has been triggered
+ Assert.assertTrue(waittingForFailOver());
+ return createFakeAllocateResponse();
+ }
+
+ }
+
+ public AllocateResponse createFakeAllocateResponse() {
+ return AllocateResponse.newInstance(-1,
+ new ArrayList<ContainerStatus>(),
+ new ArrayList<Container>(), new ArrayList<NodeReport>(),
+ Resource.newInstance(1024, 2), AMCommand.AM_RESYNC, 1,
+ null, new ArrayList<NMToken>());
+ }
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationClientProtocolOnHA.java Fri Apr 18 16:32:35 2014
@@ -51,7 +51,7 @@ public class TestApplicationClientProtoc
@Before
public void initiate() throws Exception {
- startHACluster(1, true, false);
+ startHACluster(1, true, false, false);
Configuration conf = new YarnConfiguration(this.conf);
client = createAndStartYarnClient(conf);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java Fri Apr 18 16:32:35 2014
@@ -270,9 +270,33 @@ public class TestRMFailover extends Clie
String header = getHeader("Refresh", rm2Url);
assertTrue(header.contains("; url=" + rm1Url));
+ header = getHeader("Refresh", rm2Url + "/metrics");
+ assertTrue(header.contains("; url=" + rm1Url));
+
+ header = getHeader("Refresh", rm2Url + "/jmx");
+ assertTrue(header.contains("; url=" + rm1Url));
+
+ // standby RM links /conf, /stacks, /logLevel, /static, /logs,
+ // /cluster/cluster as well as webService
+ // /ws/v1/cluster/info should not be redirected to active RM
header = getHeader("Refresh", rm2Url + "/cluster/cluster");
assertEquals(null, header);
+ header = getHeader("Refresh", rm2Url + "/conf");
+ assertEquals(null, header);
+
+ header = getHeader("Refresh", rm2Url + "/stacks");
+ assertEquals(null, header);
+
+ header = getHeader("Refresh", rm2Url + "/logLevel");
+ assertEquals(null, header);
+
+ header = getHeader("Refresh", rm2Url + "/static");
+ assertEquals(null, header);
+
+ header = getHeader("Refresh", rm2Url + "/logs");
+ assertEquals(null, header);
+
header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/info");
assertEquals(null, header);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java Fri Apr 18 16:32:35 2014
@@ -41,7 +41,7 @@ public class TestResourceTrackerOnHA ext
@Before
public void initiate() throws Exception {
- startHACluster(0, false, true);
+ startHACluster(0, false, true, false);
this.resourceTracker = getRMClient();
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Fri Apr 18 16:32:35 2014
@@ -343,9 +343,10 @@ public class TestNMClient {
// getContainerStatus can be called after stopContainer
try {
// O is possible if CLEANUP_CONTAINER is executed too late
+ // 137 is possible if the container is not terminated but killed
testGetContainerStatus(container, i, ContainerState.COMPLETE,
"Container killed by the ApplicationMaster.", Arrays.asList(
- new Integer[] {143, 0}));
+ new Integer[] {137, 143, 0}));
} catch (YarnException e) {
// The exception is possible because, after the container is stopped,
// it may be removed from NM's context.
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java Fri Apr 18 16:32:35 2014
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
/**
- * Yarn application related utilities
+ * Yarn internal application-related utilities
*/
@Private
public class Apps {
@@ -97,6 +97,17 @@ public class Apps {
}
}
}
+
+ /**
+ * This older version of this method is kept around for compatibility
+ * because downstream frameworks like Spark and Tez have been using it.
+ * Downstream frameworks are expected to move off of it.
+ */
+ @Deprecated
+ public static void setEnvFromInputString(Map<String, String> env,
+ String envString) {
+ setEnvFromInputString(env, envString, File.pathSeparator);
+ }
@Public
@Unstable
@@ -112,6 +123,18 @@ public class Apps {
environment.put(StringInterner.weakIntern(variable),
StringInterner.weakIntern(val));
}
+
+ /**
+ * This older version of this method is kept around for compatibility
+ * because downstream frameworks like Spark and Tez have been using it.
+ * Downstream frameworks are expected to move off of it.
+ */
+ @Deprecated
+ public static void addToEnvironment(
+ Map<String, String> environment,
+ String variable, String value) {
+ addToEnvironment(environment, variable, value, File.pathSeparator);
+ }
public static String crossPlatformify(String var) {
return ApplicationConstants.PARAMETER_EXPANSION_LEFT + var
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Fri Apr 18 16:32:35 2014
@@ -36,12 +36,14 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.RunJar;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -139,6 +141,16 @@ public class FSDownload implements Calla
if (!checkPublicPermsForAll(fs, sStat, FsAction.READ_EXECUTE, FsAction.READ)) {
return false;
}
+
+ if (Shell.WINDOWS && fs instanceof LocalFileSystem) {
+ // Relax the requirement for public cache on LFS on Windows since default
+ // permissions are "700" all the way up to the drive letter. In this
+ // model, the only requirement for a user is to give EVERYONE group
+ // permission on the file and the file will be considered public.
+ // This code path is only hit when fs.default.name is file:/// (mainly
+ // in tests).
+ return true;
+ }
return ancestorsHaveExecutePermissions(fs, current.getParent(), statCache);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Fri Apr 18 16:32:35 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.util;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
@@ -736,7 +735,7 @@ public class ProcfsBasedProcessTree exte
fReader = new FileReader(file);
in = new BufferedReader(fReader);
ProcessSmapMemoryInfo memoryMappingInfo = null;
- List<String> lines = IOUtils.readLines(new FileInputStream(file));
+ List<String> lines = IOUtils.readLines(in);
for (String line : lines) {
line = line.trim();
try {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java Fri Apr 18 16:32:35 2014
@@ -43,8 +43,7 @@ public class TimelineUtils {
mapper = new ObjectMapper();
AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
mapper.setAnnotationIntrospector(introspector);
- mapper.getSerializationConfig()
- .setSerializationInclusion(Inclusion.NON_NULL);
+ mapper.setSerializationInclusion(Inclusion.NON_NULL);
}
/**
@@ -77,7 +76,7 @@ public class TimelineUtils {
public static String dumpTimelineRecordtoJSON(Object o, boolean pretty)
throws JsonGenerationException, JsonMappingException, IOException {
if (pretty) {
- return mapper.defaultPrettyPrintingWriter().writeValueAsString(o);
+ return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(o);
} else {
return mapper.writeValueAsString(o);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java Fri Apr 18 16:32:35 2014
@@ -51,8 +51,7 @@ public class YarnJacksonJaxbJsonProvider
ObjectMapper mapper = super.locateMapper(type, mediaType);
AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
mapper.setAnnotationIntrospector(introspector);
- mapper.getSerializationConfig()
- .setSerializationInclusion(Inclusion.NON_NULL);
+ mapper.setSerializationInclusion(Inclusion.NON_NULL);
return mapper;
}
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Apr 18 16:32:35 2014
@@ -1019,6 +1019,19 @@
<value>500</value>
</property>
+ <property>
+ <description>Enable the node manager to recover after starting</description>
+ <name>yarn.nodemanager.recovery.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>The local filesystem directory in which the node manager will
+ store state when recovery is enabled.</description>
+ <name>yarn.nodemanager.recovery.dir</name>
+ <value>${hadoop.tmp.dir}/yarn-nm-recovery</value>
+ </property>
+
<!--Map Reduce configuration-->
<property>
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java Fri Apr 18 16:32:35 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.util;
import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES;
import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -109,19 +110,13 @@ public class TestProcfsBasedProcessTree
@Before
public void setup() throws IOException {
+ assumeTrue(Shell.LINUX);
FileContext.getLocalFSFileContext().delete(
new Path(TEST_ROOT_DIR.getAbsolutePath()), true);
}
@Test(timeout = 30000)
public void testProcessTree() throws Exception {
-
- if (!Shell.LINUX) {
- System.out
- .println("ProcfsBasedProcessTree is not available on this system. Not testing");
- return;
-
- }
try {
Assert.assertTrue(ProcfsBasedProcessTree.isAvailable());
} catch (Exception e) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Fri Apr 18 16:32:35 2014
@@ -179,7 +179,7 @@ public class FileSystemApplicationHistor
LOG.info("Completed reading history information of application " + appId);
return historyData;
} catch (IOException e) {
- LOG.error("Error when reading history file of application " + appId);
+ LOG.error("Error when reading history file of application " + appId, e);
throw e;
} finally {
hfReader.close();
@@ -296,7 +296,7 @@ public class FileSystemApplicationHistor
return historyData;
} catch (IOException e) {
LOG.error("Error when reading history file of application attempt"
- + appAttemptId);
+ + appAttemptId, e);
throw e;
} finally {
hfReader.close();
@@ -344,7 +344,7 @@ public class FileSystemApplicationHistor
+ containerId);
return historyData;
} catch (IOException e) {
- LOG.error("Error when reading history file of container " + containerId);
+ LOG.error("Error when reading history file of container " + containerId, e);
throw e;
} finally {
hfReader.close();
@@ -420,7 +420,7 @@ public class FileSystemApplicationHistor
+ appStart.getApplicationId());
} catch (IOException e) {
LOG.error("Error when openning history file of application "
- + appStart.getApplicationId());
+ + appStart.getApplicationId(), e);
throw e;
}
outstandingWriters.put(appStart.getApplicationId(), hfWriter);
@@ -437,7 +437,7 @@ public class FileSystemApplicationHistor
+ appStart.getApplicationId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing start information of application "
- + appStart.getApplicationId());
+ + appStart.getApplicationId(), e);
throw e;
}
}
@@ -456,7 +456,7 @@ public class FileSystemApplicationHistor
+ appFinish.getApplicationId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing finish information of application "
- + appFinish.getApplicationId());
+ + appFinish.getApplicationId(), e);
throw e;
} finally {
hfWriter.close();
@@ -480,7 +480,7 @@ public class FileSystemApplicationHistor
+ appAttemptStart.getApplicationAttemptId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing start information of application attempt "
- + appAttemptStart.getApplicationAttemptId());
+ + appAttemptStart.getApplicationAttemptId(), e);
throw e;
}
}
@@ -501,7 +501,7 @@ public class FileSystemApplicationHistor
+ appAttemptFinish.getApplicationAttemptId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing finish information of application attempt "
- + appAttemptFinish.getApplicationAttemptId());
+ + appAttemptFinish.getApplicationAttemptId(), e);
throw e;
}
}
@@ -521,7 +521,7 @@ public class FileSystemApplicationHistor
+ containerStart.getContainerId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing start information of container "
- + containerStart.getContainerId());
+ + containerStart.getContainerId(), e);
throw e;
}
}
@@ -541,7 +541,7 @@ public class FileSystemApplicationHistor
+ containerFinish.getContainerId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing finish information of container "
- + containerFinish.getContainerId());
+ + containerFinish.getContainerId(), e);
}
}
@@ -676,9 +676,10 @@ public class FileSystemApplicationHistor
private TFile.Reader reader;
private TFile.Reader.Scanner scanner;
+ FSDataInputStream fsdis;
public HistoryFileReader(Path historyFile) throws IOException {
- FSDataInputStream fsdis = fs.open(historyFile);
+ fsdis = fs.open(historyFile);
reader =
new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(),
getConfig());
@@ -707,7 +708,7 @@ public class FileSystemApplicationHistor
}
public void close() {
- IOUtils.cleanup(LOG, scanner, reader);
+ IOUtils.cleanup(LOG, scanner, reader, fsdis);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Fri Apr 18 16:32:35 2014
@@ -23,6 +23,8 @@ import java.net.URI;
import org.junit.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,6 +44,9 @@ import org.junit.Test;
public class TestFileSystemApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils {
+ private static Log LOG = LogFactory
+ .getLog(TestFileSystemApplicationHistoryStore.class.getName());
+
private FileSystem fs;
private Path fsWorkingPath;
@@ -50,9 +55,12 @@ public class TestFileSystemApplicationHi
fs = new RawLocalFileSystem();
Configuration conf = new Configuration();
fs.initialize(new URI("/"), conf);
- fsWorkingPath = new Path("Test");
+ fsWorkingPath =
+ new Path("target",
+ TestFileSystemApplicationHistoryStore.class.getSimpleName());
fs.delete(fsWorkingPath, true);
- conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
+ conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
+ fsWorkingPath.toString());
store = new FileSystemApplicationHistoryStore();
store.init(conf);
store.start();
@@ -67,6 +75,7 @@ public class TestFileSystemApplicationHi
@Test
public void testReadWriteHistoryData() throws IOException {
+ LOG.info("Starting testReadWriteHistoryData");
testWriteHistoryData(5);
testReadHistoryData(5);
}
@@ -167,6 +176,7 @@ public class TestFileSystemApplicationHi
@Test
public void testWriteAfterApplicationFinish() throws IOException {
+ LOG.info("Starting testWriteAfterApplicationFinish");
ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
writeApplicationFinishData(appId);
@@ -203,6 +213,7 @@ public class TestFileSystemApplicationHi
@Test
public void testMassiveWriteContainerHistoryData() throws IOException {
+ LOG.info("Starting testMassiveWriteContainerHistoryData");
long mb = 1024 * 1024;
long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
ApplicationId appId = ApplicationId.newInstance(0, 1);
@@ -221,12 +232,14 @@ public class TestFileSystemApplicationHi
@Test
public void testMissingContainerHistoryData() throws IOException {
+ LOG.info("Starting testMissingContainerHistoryData");
testWriteHistoryData(3, true, false);
testReadHistoryData(3, true, false);
}
@Test
public void testMissingApplicationAttemptHistoryData() throws IOException {
+ LOG.info("Starting testMissingApplicationAttemptHistoryData");
testWriteHistoryData(3, false, true);
testReadHistoryData(3, false, true);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Apr 18 16:32:35 2014
@@ -28,6 +28,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
@@ -127,6 +130,20 @@ public class NodeManager extends Composi
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
+ boolean recoveryEnabled = conf.getBoolean(
+ YarnConfiguration.NM_RECOVERY_ENABLED,
+ YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
+ if (recoveryEnabled) {
+ FileSystem recoveryFs = FileSystem.getLocal(conf);
+ String recoveryDirName = conf.get(YarnConfiguration.NM_RECOVERY_DIR);
+ if (recoveryDirName == null) {
+ throw new IllegalArgumentException("Recovery is enabled but " +
+ YarnConfiguration.NM_RECOVERY_DIR + " is not set.");
+ }
+ Path recoveryRoot = new Path(recoveryDirName);
+ recoveryFs.mkdirs(recoveryRoot, new FsPermission((short)0700));
+ }
+
NMContainerTokenSecretManager containerTokenSecretManager =
new NMContainerTokenSecretManager(conf);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Apr 18 16:32:35 2014
@@ -343,10 +343,6 @@ public class NodeStatusUpdaterImpl exten
+ ", " + nodeHealthStatus.getHealthReport());
}
List<ContainerStatus> containersStatuses = getContainerStatuses();
- if (LOG.isDebugEnabled()) {
- LOG.debug(this.nodeId + " sending out status for "
- + containersStatuses.size() + " containers");
- }
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
createKeepAliveApplicationList(), nodeHealthStatus);
@@ -373,7 +369,8 @@ public class NodeStatusUpdaterImpl exten
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Sending out container statuses: " + containerStatuses);
+ LOG.debug("Sending out " + containerStatuses.size()
+ + " container statuses: " + containerStatuses);
}
return containerStatuses;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java Fri Apr 18 16:32:35 2014
@@ -29,15 +29,18 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.service.ServiceStateChangeListener;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
-import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
@@ -46,6 +49,8 @@ import com.google.common.base.Preconditi
public class AuxServices extends AbstractService
implements ServiceStateChangeListener, EventHandler<AuxServicesEvent> {
+ static final String STATE_STORE_ROOT_NAME = "nm-aux-services";
+
private static final Log LOG = LogFactory.getLog(AuxServices.class);
protected final Map<String,AuxiliaryService> serviceMap;
@@ -91,6 +96,17 @@ public class AuxServices extends Abstrac
@Override
public void serviceInit(Configuration conf) throws Exception {
+ final FsPermission storeDirPerms = new FsPermission((short)0700);
+ Path stateStoreRoot = null;
+ FileSystem stateStoreFs = null;
+ boolean recoveryEnabled = conf.getBoolean(
+ YarnConfiguration.NM_RECOVERY_ENABLED,
+ YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
+ if (recoveryEnabled) {
+ stateStoreRoot = new Path(conf.get(YarnConfiguration.NM_RECOVERY_DIR),
+ STATE_STORE_ROOT_NAME);
+ stateStoreFs = FileSystem.getLocal(conf);
+ }
Collection<String> auxNames = conf.getStringCollection(
YarnConfiguration.NM_AUX_SERVICES);
for (final String sName : auxNames) {
@@ -119,6 +135,11 @@ public class AuxServices extends Abstrac
+"the name in the config.");
}
addService(sName, s);
+ if (recoveryEnabled) {
+ Path storePath = new Path(stateStoreRoot, sName);
+ stateStoreFs.mkdirs(storePath, storeDirPerms);
+ s.setRecoveryPath(storePath);
+ }
s.init(conf);
} catch (RuntimeException e) {
LOG.fatal("Failed to initialize " + sName, e);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri Apr 18 16:32:35 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -141,7 +142,7 @@ public class ContainerImpl implements Co
ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.NEW, ContainerState.DONE,
- ContainerEventType.KILL_CONTAINER, CONTAINER_DONE_TRANSITION)
+ ContainerEventType.KILL_CONTAINER, new KillOnNewTransition())
// From LOCALIZING State
.addTransition(ContainerState.LOCALIZING,
@@ -760,7 +761,9 @@ public class ContainerImpl implements Co
container.cleanup();
container.metrics.endInitingContainer();
ContainerKillEvent killEvent = (ContainerKillEvent) event;
+ container.exitCode = ExitCode.TERMINATED.getExitCode();
container.diagnostics.append(killEvent.getDiagnostic()).append("\n");
+ container.diagnostics.append("Container is killed before being launched.\n");
}
}
@@ -828,7 +831,6 @@ public class ContainerImpl implements Co
/**
* Handle the following transitions:
- * - NEW -> DONE upon KILL_CONTAINER
* - {LOCALIZATION_FAILED, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE,
* KILLING, CONTAINER_CLEANEDUP_AFTER_KILL}
* -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
@@ -850,6 +852,21 @@ public class ContainerImpl implements Co
}
/**
+ * Handle the following transition:
+ * - NEW -> DONE upon KILL_CONTAINER
+ */
+ static class KillOnNewTransition extends ContainerDoneTransition {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerKillEvent killEvent = (ContainerKillEvent) event;
+ container.exitCode = ExitCode.TERMINATED.getExitCode();
+ container.diagnostics.append(killEvent.getDiagnostic()).append("\n");
+ container.diagnostics.append("Container is killed before being launched.\n");
+ super.transition(container, event);
+ }
+ }
+
+ /**
* Update diagnostics, staying in the same state.
*/
static class ContainerDiagnosticsUpdateTransition implements
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java Fri Apr 18 16:32:35 2014
@@ -78,11 +78,17 @@ public class TestNodeHealthService {
}
private void writeNodeHealthScriptFile(String scriptStr, boolean setExecutable)
- throws IOException {
- PrintWriter pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile));
- pw.println(scriptStr);
- pw.flush();
- pw.close();
+ throws IOException {
+ PrintWriter pw = null;
+ try {
+ FileUtil.setWritable(nodeHealthscriptFile, true);
+ FileUtil.setReadable(nodeHealthscriptFile, true);
+ pw = new PrintWriter(new FileOutputStream(nodeHealthscriptFile));
+ pw.println(scriptStr);
+ pw.flush();
+ } finally {
+ pw.close();
+ }
FileUtil.setExecutable(nodeHealthscriptFile, setExecutable);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri Apr 18 16:32:35 2014
@@ -261,10 +261,12 @@ public class TestNodeStatusUpdater {
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
- Assert.assertEquals("Number of applications should only be one!", 1,
+ Assert.assertEquals("Number of applications should have two!", 2,
appToContainers.size());
- Assert.assertEquals("Number of container for the app should be two!",
- 2, appToContainers.get(appId2).size());
+ Assert.assertEquals("Number of container for the app-1 should be only one!",
+ 1, appToContainers.get(appId1).size());
+ Assert.assertEquals("Number of container for the app-2 should be only one!",
+ 1, appToContainers.get(appId2).size());
// Checks on the NM end
ConcurrentMap<ContainerId, Container> activeContainers =