You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2013/12/20 02:01:26 UTC
svn commit: r1552467 [1/2] - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/j...
Author: cnauroth
Date: Fri Dec 20 01:01:18 2013
New Revision: 1552467
URL: http://svn.apache.org/r1552467
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeResourceRequest.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeResourceRequest.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeResourceResponse.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeResourceResponse.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMFailoverProxyProvider.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMFailoverProxyProvider.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceResponsePBImpl.java
- copied unchanged from r1552465, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceResponsePBImpl.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceManagerAdministrationProtocolPBClientImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationResponsePBImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/pom.xml
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt Fri Dec 20 01:01:18 2013
@@ -49,6 +49,12 @@ Release 2.4.0 - UNRELEASED
YARN-1448. AM-RM protocol changes to support container resizing (Wangda Tan
via Sandy Ryza)
+ YARN-312. Introduced ResourceManagerAdministrationProtocol changes to support
+ changing resources on node. (Junping Du via vinodkv)
+
+ YARN-1028. Added FailoverProxyProvider capability to ResourceManager to help
+ with RM failover. (Karthik Kambatla via vinodkv)
+
IMPROVEMENTS
YARN-7. Support CPU resource for DistributedShell. (Junping Du via llu)
@@ -175,6 +181,12 @@ Release 2.4.0 - UNRELEASED
YARN-1435. Modified Distributed Shell to accept either the command or the
custom script. (Xuan Gong via zjshen)
+ YARN-1446. Changed client API to retry killing application till RM
+ acknowledges so as to account for RM crashes/failover. (Jian He via vinodkv)
+
+ YARN-1307. Redesign znode structure for Zookeeper based RM state-store for
+ better organization and scalability. (Tsuyoshi OZAWA via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -252,6 +264,9 @@ Release 2.4.0 - UNRELEASED
YARN-1145. Fixed a potential file-handle leak in the web interface for
displaying aggregated logs. (Rohith Sharma via vinodkv)
+ YARN-1451. TestResourceManager relies on the scheduler assigning multiple
+ containers in a single node update. (Sandy Ryza via kasha)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Fri Dec 20 01:01:18 2013
@@ -310,4 +310,12 @@
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Ignore INSTANCE not being final as it is created in sub-classes -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.client.RMProxy" />
+ <Field name="INSTANCE" />
+ <Bug pattern="MS_SHOULD_BE_FINAL"/>
+ </Match>
+
+
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java Fri Dec 20 01:01:18 2013
@@ -26,10 +26,21 @@ import org.apache.hadoop.yarn.api.Applic
import org.apache.hadoop.yarn.util.Records;
/**
- * <p>The response sent by the <code>ResourceManager</code> to the client
- * aborting a submitted application.</p>
- *
- * <p>Currently it's empty.</p>
+ * <p>
+ * The response sent by the <code>ResourceManager</code> to the client aborting
+ * a submitted application.
+ * </p>
+ * <p>
+ * The response, includes:
+ * <ul>
+ * <li>A flag which indicates that the process of killing the application is
+ * completed or not.</li>
+ * </ul>
+ * Note: user is recommended to wait until this flag becomes true, otherwise if
+ * the <code>ResourceManager</code> crashes before the process of killing the
+ * application is completed, the <code>ResourceManager</code> may retry this
+ * application on recovery.
+ * </p>
*
* @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)
*/
@@ -38,9 +49,24 @@ import org.apache.hadoop.yarn.util.Recor
public abstract class KillApplicationResponse {
@Private
@Unstable
- public static KillApplicationResponse newInstance() {
+ public static KillApplicationResponse newInstance(boolean isKillCompleted) {
KillApplicationResponse response =
Records.newRecord(KillApplicationResponse.class);
+ response.setIsKillCompleted(isKillCompleted);
return response;
}
+
+ /**
+ * Get the flag which indicates that the process of killing application is completed or not.
+ */
+ @Public
+ @Stable
+ public abstract boolean getIsKillCompleted();
+
+ /**
+ * Set the flag which indicates that the process of killing application is completed or not.
+ */
+ @Private
+ @Unstable
+ public abstract void setIsKillCompleted(boolean isKillCompleted);
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java Fri Dec 20 01:01:18 2013
@@ -62,4 +62,10 @@ public abstract class ResourceOption {
protected abstract void build();
+ @Override
+ public String toString() {
+ return "Resource:" + getResource().toString()
+ + ", overCommitTimeout:" + getOverCommitTimeout();
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Dec 20 01:01:18 2013
@@ -27,7 +27,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
@@ -297,6 +296,31 @@ public class YarnConfiguration extends C
HttpConfig.isSecure() ? RM_WEBAPP_HTTPS_ADDRESS
: RM_WEBAPP_ADDRESS));
+ public static final String CLIENT_FAILOVER_PREFIX =
+ YARN_PREFIX + "client.failover-";
+ public static final String CLIENT_FAILOVER_PROXY_PROVIDER =
+ CLIENT_FAILOVER_PREFIX + "proxy-provider";
+ public static final String DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER =
+ "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider";
+
+ public static final String CLIENT_FAILOVER_MAX_ATTEMPTS =
+ CLIENT_FAILOVER_PREFIX + "max-attempts";
+
+ public static final String CLIENT_FAILOVER_SLEEPTIME_BASE_MS =
+ CLIENT_FAILOVER_PREFIX + "sleep-base-ms";
+
+ public static final String CLIENT_FAILOVER_SLEEPTIME_MAX_MS =
+ CLIENT_FAILOVER_PREFIX + "sleep-max-ms";
+
+ public static final String CLIENT_FAILOVER_RETRIES =
+ CLIENT_FAILOVER_PREFIX + "retries";
+ public static final int DEFAULT_CLIENT_FAILOVER_RETRIES = 0;
+
+ public static final String CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS =
+ CLIENT_FAILOVER_PREFIX + "retries-on-socket-timeouts";
+ public static final int
+ DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0;
+
////////////////////////////////
// RM state store configs
////////////////////////////////
@@ -851,22 +875,31 @@ public class YarnConfiguration extends C
public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX
+ "is.minicluster";
+ public static final String YARN_MC_PREFIX = YARN_PREFIX + "minicluster.";
+
/** Whether to use fixed ports with the minicluster. */
- public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX
- + "minicluster.fixed.ports";
+ public static final String YARN_MINICLUSTER_FIXED_PORTS =
+ YARN_MC_PREFIX + "fixed.ports";
/**
* Default is false to be able to run tests concurrently without port
* conflicts.
*/
- public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
+ public static final boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
+
+ /**
+ * Whether the NM should use RPC to connect to the RM. Default is false.
+ * Can be set to true only when using fixed ports.
+ */
+ public static final String YARN_MINICLUSTER_USE_RPC = YARN_MC_PREFIX + "use-rpc";
+ public static final boolean DEFAULT_YARN_MINICLUSTER_USE_RPC = false;
/**
* Whether users are explicitly trying to control resource monitoring
* configuration for the MiniYARNCluster. Disabled by default.
*/
public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING =
- YARN_PREFIX + "minicluster.control-resource-monitoring";
+ YARN_MC_PREFIX + "control-resource-monitoring";
public static final boolean
DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false;
@@ -882,15 +915,23 @@ public class YarnConfiguration extends C
////////////////////////////////
/**
+ * Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
* The interval of the yarn client's querying application state after
* application submission. The unit is millisecond.
*/
+ @Deprecated
public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
YARN_PREFIX + "client.app-submission.poll-interval";
- public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
- 1000;
/**
+ * The interval that the yarn client library uses to poll the completion
+ * status of the asynchronous API of application client protocol.
+ */
+ public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS =
+ YARN_PREFIX + "client.application-client-protocol.poll-interval-ms";
+ public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS =
+ 200;
+ /**
* Max number of threads in NMClientAsync to process container management
* events
*/
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java Fri Dec 20 01:01:18 2013
@@ -22,8 +22,11 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.exceptions.RMNotYetActiveException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
@@ -38,6 +41,8 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
@Private
@Stable
@@ -77,4 +82,24 @@ public interface ResourceManagerAdminist
public RefreshServiceAclsResponse refreshServiceAcls(
RefreshServiceAclsRequest request)
throws YarnException, IOException;
+
+ /**
+ * <p>The interface used by admin to update nodes' resources to the
+ * <code>ResourceManager</code> </p>.
+ *
+ * <p>The admin client is required to provide details such as a map from
+ * {@link NodeId} to {@link ResourceOption} required to update resources on
+ * a list of <code>RMNode</code> in <code>ResourceManager</code> etc.
+ * via the {@link UpdateNodeResourceRequest}.</p>
+ *
+ * @param request request to update resource for a node in cluster.
+ * @return (empty) response on accepting update.
+ * @throws YarnException
+ * @throws IOException
+ */
+ @Public
+ @Evolving
+ public UpdateNodeResourceResponse updateNodeResource(
+ UpdateNodeResourceRequest request)
+ throws YarnException, IOException;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto Fri Dec 20 01:01:18 2013
@@ -38,4 +38,5 @@ service ResourceManagerAdministrationPro
rpc refreshAdminAcls(RefreshAdminAclsRequestProto) returns (RefreshAdminAclsResponseProto);
rpc refreshServiceAcls(RefreshServiceAclsRequestProto) returns (RefreshServiceAclsResponseProto);
rpc getGroupsForUser(GetGroupsForUserRequestProto) returns (GetGroupsForUserResponseProto);
+ rpc updateNodeResource (UpdateNodeResourceRequestProto) returns (UpdateNodeResourceResponseProto);
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Fri Dec 20 01:01:18 2013
@@ -68,6 +68,13 @@ message GetGroupsForUserResponseProto {
repeated string groups = 1;
}
+message UpdateNodeResourceRequestProto {
+ repeated NodeResourceMapProto node_resource_map = 1;
+}
+
+message UpdateNodeResourceResponseProto {
+}
+
////////////////////////////////////////////////////////////////////////
////// RM recovery related records /////////////////////////////////////
////////////////////////////////////////////////////////////////////////
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Dec 20 01:01:18 2013
@@ -63,6 +63,11 @@ message ResourceOptionProto {
optional int32 over_commit_timeout = 2;
}
+message NodeResourceMapProto {
+ optional NodeIdProto node_id = 1;
+ optional ResourceOptionProto resource_option = 2;
+}
+
message PriorityProto {
optional int32 priority = 1;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Fri Dec 20 01:01:18 2013
@@ -116,6 +116,7 @@ message KillApplicationRequestProto {
}
message KillApplicationResponseProto {
+ optional bool is_kill_completed = 1 [default = false];
}
message GetClusterMetricsRequestProto {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java Fri Dec 20 01:01:18 2013
@@ -23,6 +23,7 @@ import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -34,17 +35,37 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
-public class ClientRMProxy<T> extends RMProxy<T> {
+import com.google.common.base.Preconditions;
+public class ClientRMProxy<T> extends RMProxy<T> {
private static final Log LOG = LogFactory.getLog(ClientRMProxy.class);
+ private interface ClientRMProtocols extends ApplicationClientProtocol,
+ ApplicationMasterProtocol, ResourceManagerAdministrationProtocol {
+ // Add nothing
+ }
+
+ static {
+ INSTANCE = new ClientRMProxy();
+ }
+
+ private ClientRMProxy(){
+ super();
+ }
+
+ /**
+ * Create a proxy to the ResourceManager for the specified protocol.
+ * @param configuration Configuration with all the required information.
+ * @param protocol Client protocol for which proxy is being requested.
+ * @param <T> Type of proxy.
+ * @return Proxy to the ResourceManager for the specified client protocol.
+ * @throws IOException
+ */
public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
- YarnConfiguration conf = (configuration instanceof YarnConfiguration)
- ? (YarnConfiguration) configuration
- : new YarnConfiguration(configuration);
- InetSocketAddress rmAddress = getRMAddress(conf, protocol);
- return createRMProxy(conf, protocol, rmAddress);
+ // This method exists only to initiate this class' static INSTANCE. TODO:
+ // FIX if possible
+ return RMProxy.createRMProxy(configuration, protocol);
}
private static void setupTokens(InetSocketAddress resourceManagerAddress)
@@ -63,7 +84,9 @@ public class ClientRMProxy<T> extends RM
}
}
- private static InetSocketAddress getRMAddress(YarnConfiguration conf,
+ @InterfaceAudience.Private
+ @Override
+ protected InetSocketAddress getRMAddress(YarnConfiguration conf,
Class<?> protocol) throws IOException {
if (protocol == ApplicationClientProtocol.class) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
@@ -89,4 +112,12 @@ public class ClientRMProxy<T> extends RM
throw new IllegalStateException(message);
}
}
+
+ @InterfaceAudience.Private
+ @Override
+ protected void checkAllowedProtocols(Class<?> protocol) {
+ Preconditions.checkArgument(
+ protocol.isAssignableFrom(ClientRMProtocols.class),
+ "RM does not support this client protocol");
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Fri Dec 20 01:01:18 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -48,6 +47,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -78,8 +78,8 @@ public class YarnClientImpl extends Yarn
private static final Log LOG = LogFactory.getLog(YarnClientImpl.class);
protected ApplicationClientProtocol rmClient;
- protected InetSocketAddress rmAddress;
- protected long statePollIntervalMillis;
+ protected long submitPollIntervalMillis;
+ private long asyncApiPollIntervalMillis;
private static final String ROOT = "root";
@@ -87,17 +87,19 @@ public class YarnClientImpl extends Yarn
super(YarnClientImpl.class.getName());
}
- private static InetSocketAddress getRmAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
- }
-
+ @SuppressWarnings("deprecation")
@Override
protected void serviceInit(Configuration conf) throws Exception {
- this.rmAddress = getRmAddress(conf);
- statePollIntervalMillis = conf.getLong(
+ asyncApiPollIntervalMillis =
+ conf.getLong(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
+ submitPollIntervalMillis = asyncApiPollIntervalMillis;
+ if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS)
+ != null) {
+ submitPollIntervalMillis = conf.getLong(
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
- YarnConfiguration.DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
+ YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
+ }
super.serviceInit(conf);
}
@@ -165,25 +167,41 @@ public class YarnClientImpl extends Yarn
" is still in " + state);
}
try {
- Thread.sleep(statePollIntervalMillis);
+ Thread.sleep(submitPollIntervalMillis);
} catch (InterruptedException ie) {
}
}
-
- LOG.info("Submitted application " + applicationId + " to ResourceManager"
- + " at " + rmAddress);
+ LOG.info("Submitted application " + applicationId);
return applicationId;
}
@Override
public void killApplication(ApplicationId applicationId)
throws YarnException, IOException {
- LOG.info("Killing application " + applicationId);
KillApplicationRequest request =
Records.newRecord(KillApplicationRequest.class);
request.setApplicationId(applicationId);
- rmClient.forceKillApplication(request);
+
+ try {
+ int pollCount = 0;
+ while (true) {
+ KillApplicationResponse response =
+ rmClient.forceKillApplication(request);
+ if (response.getIsKillCompleted()) {
+ break;
+ }
+ if (++pollCount % 10 == 0) {
+ LOG.info("Watiting for application " + applicationId
+ + " to be killed.");
+ }
+ Thread.sleep(asyncApiPollIntervalMillis);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while waiting for application " + applicationId
+ + " to be killed.");
+ }
+ LOG.info("Killed application " + applicationId);
}
@Override
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceManagerAdministrationProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceManagerAdministrationProtocolPBClientImpl.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceManagerAdministrationProtocolPBClientImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceManagerAdministrationProtocolPBClientImpl.java Fri Dec 20 01:01:18 2013
@@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -159,6 +161,14 @@ public class TestResourceManagerAdminist
RefreshAdminAclsResponse response = client.refreshAdminAcls(request);
assertNotNull(response);
}
+
+ @Test
+ public void testUpdateNodeResource() throws Exception {
+ UpdateNodeResourceRequest request = recordFactory
+ .newRecordInstance(UpdateNodeResourceRequest.class);
+ UpdateNodeResourceResponse response = client.updateNodeResource(request);
+ assertNotNull(response);
+ }
@Test
public void testRefreshServiceAcls() throws Exception {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Fri Dec 20 01:01:18 2013
@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -89,6 +91,7 @@ public class TestYarnClient {
rm.stop();
}
+ @SuppressWarnings("deprecation")
@Test (timeout = 30000)
public void testSubmitApplication() {
Configuration conf = new Configuration();
@@ -128,6 +131,23 @@ public class TestYarnClient {
client.stop();
}
+ @Test
+ public void testKillApplication() throws Exception {
+ MockRM rm = new MockRM();
+ rm.start();
+ RMApp app = rm.submitApp(2000);
+
+ Configuration conf = new Configuration();
+ @SuppressWarnings("resource")
+ final YarnClient client = new MockYarnClient();
+ client.init(conf);
+ client.start();
+
+ client.killApplication(app.getApplicationId());
+ verify(((MockYarnClient) client).getRMClient(), times(2))
+ .forceKillApplication(any(KillApplicationRequest.class));
+ }
+
@Test(timeout = 30000)
public void testApplicationType() throws Exception {
Logger rootLogger = LogManager.getRootLogger();
@@ -234,6 +254,11 @@ public class TestYarnClient {
GetApplicationReportRequest.class))).thenReturn(mockResponse);
when(rmClient.getApplications(any(GetApplicationsRequest.class)))
.thenReturn(mockAppResponse);
+ // return false for 1st kill request, and true for the 2nd.
+ when(rmClient.forceKillApplication(any(
+ KillApplicationRequest.class)))
+ .thenReturn(KillApplicationResponse.newInstance(false)).thenReturn(
+ KillApplicationResponse.newInstance(true));
} catch (YarnException e) {
Assert.fail("Exception is not expected.");
} catch (IOException e) {
@@ -242,6 +267,10 @@ public class TestYarnClient {
when(mockResponse.getApplicationReport()).thenReturn(mockReport);
}
+ public ApplicationClientProtocol getRMClient() {
+ return rmClient;
+ }
+
@Override
public List<ApplicationReport> getApplications(
Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates)
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationResponsePBImpl.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationResponsePBImpl.java Fri Dec 20 01:01:18 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationResponseProtoOrBuilder;
import com.google.protobuf.TextFormat;
@@ -67,4 +68,24 @@ public class KillApplicationResponsePBIm
public String toString() {
return TextFormat.shortDebugString(getProto());
}
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = KillApplicationResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public boolean getIsKillCompleted() {
+ KillApplicationResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getIsKillCompleted();
+ }
+
+ @Override
+ public void setIsKillCompleted(boolean isKillCompleted) {
+ maybeInitBuilder();
+ builder.setIsKillCompleted(isKillCompleted);
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java Fri Dec 20 01:01:18 2013
@@ -36,6 +36,8 @@ import org.apache.hadoop.io.retry.RetryP
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -48,7 +50,68 @@ import com.google.common.annotations.Vis
public class RMProxy<T> {
private static final Log LOG = LogFactory.getLog(RMProxy.class);
+ protected static RMProxy INSTANCE;
+ protected RMProxy() {}
+
+ /**
+ * Verify the passed protocol is supported.
+ */
+ @Private
+ protected void checkAllowedProtocols(Class<?> protocol) {}
+
+ /**
+ * Get the ResourceManager address from the provided Configuration for the
+ * given protocol.
+ */
+ @Private
+ protected InetSocketAddress getRMAddress(
+ YarnConfiguration conf, Class<?> protocol) throws IOException {
+ throw new UnsupportedOperationException("This method should be invoked " +
+ "from an instance of ClientRMProxy or ServerRMProxy");
+ }
+
+ /**
+ * Create a proxy for the specified protocol. For non-HA,
+ * this is a direct connection to the ResourceManager address. When HA is
+ * enabled, the proxy handles the failover between the ResourceManagers as
+ * well.
+ */
+ @Private
+ protected static <T> T createRMProxy(final Configuration configuration,
+ final Class<T> protocol) throws IOException {
+ YarnConfiguration conf = (configuration instanceof YarnConfiguration)
+ ? (YarnConfiguration) configuration
+ : new YarnConfiguration(configuration);
+ RetryPolicy retryPolicy = createRetryPolicy(conf);
+ if (HAUtil.isHAEnabled(conf)) {
+ RMFailoverProxyProvider<T> provider =
+ INSTANCE.createRMFailoverProxyProvider(conf, protocol);
+ return (T) RetryProxy.create(protocol, provider, retryPolicy);
+ } else {
+ InetSocketAddress rmAddress = INSTANCE.getRMAddress(conf, protocol);
+ LOG.info("Connecting to ResourceManager at " + rmAddress);
+ T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
+ return (T) RetryProxy.create(protocol, proxy, retryPolicy);
+ }
+ }
+
+ /**
+ * @deprecated
+ * This method is deprecated and is not used by YARN internally any more.
+ * To create a proxy to the RM, use ClientRMProxy#createRMProxy or
+ * ServerRMProxy#createRMProxy.
+ *
+ * Create a proxy to the ResourceManager at the specified address.
+ *
+ * @param conf Configuration to generate retry policy
+ * @param protocol Protocol for the proxy
+ * @param rmAddress Address of the ResourceManager
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException
+ */
+ @Deprecated
public static <T> T createRMProxy(final Configuration conf,
final Class<T> protocol, InetSocketAddress rmAddress) throws IOException {
RetryPolicy retryPolicy = createRetryPolicy(conf);
@@ -57,12 +120,16 @@ public class RMProxy<T> {
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
}
- private static <T> T getProxy(final Configuration conf,
+ /**
+ * Get a proxy to the RM at the specified address. To be used to create a
+ * RetryProxy.
+ */
+ @Private
+ static <T> T getProxy(final Configuration conf,
final Class<T> protocol, final InetSocketAddress rmAddress)
throws IOException {
return UserGroupInformation.getCurrentUser().doAs(
new PrivilegedAction<T>() {
-
@Override
public T run() {
return (T) YarnRPC.create(conf).getProxy(protocol, rmAddress, conf);
@@ -70,6 +137,50 @@ public class RMProxy<T> {
});
}
+ /**
+ * Helper method to create FailoverProxyProvider.
+ */
+ private <T> RMFailoverProxyProvider<T> createRMFailoverProxyProvider(
+ Configuration conf, Class<T> protocol) {
+ Class<? extends RMFailoverProxyProvider<T>> defaultProviderClass;
+ try {
+ defaultProviderClass = (Class<? extends RMFailoverProxyProvider<T>>)
+ Class.forName(
+ YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER);
+ } catch (Exception e) {
+ throw new YarnRuntimeException("Invalid default failover provider class" +
+ YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER, e);
+ }
+
+ RMFailoverProxyProvider<T> provider = ReflectionUtils.newInstance(
+ conf.getClass(YarnConfiguration.CLIENT_FAILOVER_PROXY_PROVIDER,
+ defaultProviderClass, RMFailoverProxyProvider.class), conf);
+ provider.init(conf, (RMProxy<T>) this, protocol);
+ return provider;
+ }
+
+ /**
+ * A RetryPolicy to allow failing over upto the specified maximum time.
+ */
+ private static class FailoverUptoMaximumTimePolicy implements RetryPolicy {
+ private long maxTime;
+
+ FailoverUptoMaximumTimePolicy(long maxTime) {
+ this.maxTime = maxTime;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e, int retries, int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ return System.currentTimeMillis() < maxTime
+ ? RetryAction.FAILOVER_AND_RETRY
+ : RetryAction.FAIL;
+ }
+ }
+
+ /**
+ * Fetch retry policy from Configuration
+ */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
@@ -81,19 +192,10 @@ public class RMProxy<T> {
conf.getLong(
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
YarnConfiguration
- .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);
-
- if (rmConnectionRetryIntervalMS < 0) {
- throw new YarnRuntimeException("Invalid Configuration. " +
- YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
- " should not be negative.");
- }
+ .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);
boolean waitForEver = (rmConnectWaitMS == -1);
-
- if (waitForEver) {
- return RetryPolicies.RETRY_FOREVER;
- } else {
+ if (!waitForEver) {
if (rmConnectWaitMS < 0) {
throw new YarnRuntimeException("Invalid Configuration. "
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
@@ -110,18 +212,54 @@ public class RMProxy<T> {
}
}
+ // Handle HA case first
+ if (HAUtil.isHAEnabled(conf)) {
+ final long failoverSleepBaseMs = conf.getLong(
+ YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
+ rmConnectionRetryIntervalMS);
+
+ final long failoverSleepMaxMs = conf.getLong(
+ YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
+ rmConnectionRetryIntervalMS);
+
+ int maxFailoverAttempts = conf.getInt(
+ YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);
+
+ RetryPolicy basePolicy = RetryPolicies.TRY_ONCE_THEN_FAIL;
+ if (maxFailoverAttempts == -1) {
+ if (waitForEver) {
+ basePolicy = RetryPolicies.FAILOVER_FOREVER;
+ } else {
+ basePolicy = new FailoverUptoMaximumTimePolicy(
+ System.currentTimeMillis() + rmConnectWaitMS);
+ }
+ maxFailoverAttempts = 0;
+ }
+
+ return RetryPolicies.failoverOnNetworkException(basePolicy,
+ maxFailoverAttempts, failoverSleepBaseMs, failoverSleepMaxMs);
+ }
+
+ if (waitForEver) {
+ return RetryPolicies.RETRY_FOREVER;
+ }
+
+ if (rmConnectionRetryIntervalMS < 0) {
+ throw new YarnRuntimeException("Invalid Configuration. " +
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
+ " should not be negative.");
+ }
+
RetryPolicy retryPolicy =
RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
- rmConnectionRetryIntervalMS,
- TimeUnit.MILLISECONDS);
+ rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
new HashMap<Class<? extends Exception>, RetryPolicy>();
exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
//TO DO: after HADOOP-9576, IOException can be changed to EOFException
exceptionToPolicyMap.put(IOException.class, retryPolicy);
-
- return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
- exceptionToPolicyMap);
+ return RetryPolicies.retryByException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java Fri Dec 20 01:01:18 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshServiceAclsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
@@ -51,6 +52,8 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl;
@@ -63,6 +66,8 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl;
import com.google.protobuf.ServiceException;
@@ -186,5 +191,19 @@ public class ResourceManagerAdministrati
throw ProtobufHelper.getRemoteException(e);
}
}
+
+ @Override
+ public UpdateNodeResourceResponse updateNodeResource(
+ UpdateNodeResourceRequest request) throws YarnException, IOException {
+ UpdateNodeResourceRequestProto requestProto =
+ ((UpdateNodeResourceRequestPBImpl) request).getProto();
+ try {
+ return new UpdateNodeResourceResponsePBImpl(proxy.updateNodeResource(null,
+ requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java Fri Dec 20 01:01:18 2013
@@ -36,6 +36,8 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
@@ -44,6 +46,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl;
@@ -56,6 +59,8 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -183,5 +188,20 @@ public class ResourceManagerAdministrati
throw new ServiceException(e);
}
}
+
+ @Override
+ public UpdateNodeResourceResponseProto updateNodeResource(RpcController controller,
+ UpdateNodeResourceRequestProto proto) throws ServiceException {
+ UpdateNodeResourceRequestPBImpl request =
+ new UpdateNodeResourceRequestPBImpl(proto);
+ try {
+ UpdateNodeResourceResponse response = real.updateNodeResource(request);
+ return ((UpdateNodeResourceResponsePBImpl)response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Dec 20 01:01:18 2013
@@ -425,6 +425,61 @@
</property>
<property>
+ <description>When HA is enabled, the class to be used by Clients, AMs and
+ NMs to failover to the Active RM. It should extend
+ org.apache.hadoop.yarn.client.RMFailoverProxyProvider</description>
+ <name>yarn.client.failover-proxy-provider</name>
+ <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>
+ </property>
+
+ <property>
+ <description>When HA is enabled, the max number of times
+ FailoverProxyProvider should attempt failover. When set,
+ this overrides the yarn.resourcemanager.connect.max-wait.ms. When
+ not set, this is inferred from
+ yarn.resourcemanager.connect.max-wait.ms.</description>
+ <name>yarn.client.failover-max-attempts</name>
+ <!--value>15</value-->
+ </property>
+
+ <property>
+ <description>When HA is enabled, the sleep base (in milliseconds) to be
+ used for calculating the exponential delay between failovers. When set,
+ this overrides the yarn.resourcemanager.connect.* settings. When
+ not set, yarn.resourcemanager.connect.retry-interval.ms is used instead.
+ </description>
+ <name>yarn.client.failover-sleep-base-ms</name>
+ <!--value>500</value-->
+ </property>
+
+ <property>
+ <description>When HA is enabled, the maximum sleep time (in milliseconds)
+ between failovers. When set, this overrides the
+ yarn.resourcemanager.connect.* settings. When not set,
+ yarn.resourcemanager.connect.retry-interval.ms is used instead.</description>
+ <name>yarn.client.failover-sleep-max-ms</name>
+ <!--value>15000</value-->
+ </property>
+
+ <property>
+ <description>When HA is enabled, the number of retries per
+ attempt to connect to a ResourceManager. In other words,
+ it is the ipc.client.connect.max.retries to be used during
+ failover attempts</description>
+ <name>yarn.client.failover-retries</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <description>When HA is enabled, the number of retries per
+ attempt to connect to a ResourceManager on socket timeouts. In other
+ words, it is the ipc.client.connect.max.retries.on.timeouts to be used
+ during failover attempts</description>
+ <name>yarn.client.failover-retries-on-socket-timeouts</name>
+ <value>0</value>
+ </property>
+
+ <property>
<description>The maximum number of completed applications RM keeps. </description>
<name>yarn.resourcemanager.max-completed-applications</name>
<value>10000</value>
@@ -945,10 +1000,10 @@
<!-- Other configuration -->
<property>
- <description>The interval of the yarn client's querying application state
- after application submission. The unit is millisecond.</description>
- <name>yarn.client.app-submission.poll-interval</name>
- <value>1000</value>
+ <description>The interval that the yarn client library uses to poll the
+ completion status of the asynchronous API of application client protocol.
+ </description>
+ <name>yarn.client.application-client-protocol.poll-interval-ms</name>
+ <value>200</value>
</property>
-
</configuration>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java Fri Dec 20 01:01:18 2013
@@ -23,25 +23,43 @@ import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.client.RMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-public class ServerRMProxy<T> extends RMProxy<T> {
+import com.google.common.base.Preconditions;
+public class ServerRMProxy<T> extends RMProxy<T> {
private static final Log LOG = LogFactory.getLog(ServerRMProxy.class);
+ static {
+ INSTANCE = new ServerRMProxy();
+ }
+
+ private ServerRMProxy() {
+ super();
+ }
+
+ /**
+ * Create a proxy to the ResourceManager for the specified protocol.
+ * @param configuration Configuration with all the required information.
+ * @param protocol Server protocol for which proxy is being requested.
+ * @param <T> Type of proxy.
+ * @return Proxy to the ResourceManager for the specified server protocol.
+ * @throws IOException
+ */
public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
- YarnConfiguration conf = (configuration instanceof YarnConfiguration)
- ? (YarnConfiguration) configuration
- : new YarnConfiguration(configuration);
- InetSocketAddress rmAddress = getRMAddress(conf, protocol);
- return createRMProxy(conf, protocol, rmAddress);
+ // This method exists only to initiate this class' static INSTANCE. TODO:
+ // FIX if possible
+ return RMProxy.createRMProxy(configuration, protocol);
}
- private static InetSocketAddress getRMAddress(YarnConfiguration conf,
- Class<?> protocol) {
+ @InterfaceAudience.Private
+ @Override
+ protected InetSocketAddress getRMAddress(YarnConfiguration conf,
+ Class<?> protocol) {
if (protocol == ResourceTracker.class) {
return conf.getSocketAddr(
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
@@ -55,4 +73,12 @@ public class ServerRMProxy<T> extends RM
throw new IllegalStateException(message);
}
}
+
+ @InterfaceAudience.Private
+ @Override
+ protected void checkAllowedProtocols(Class<?> protocol) {
+ Preconditions.checkArgument(
+ protocol.isAssignableFrom(ResourceTracker.class),
+ "ResourceManager does not support this protocol");
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Fri Dec 20 01:01:18 2013
@@ -20,6 +20,11 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -41,6 +46,8 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.RMNotYetActiveException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -61,6 +68,9 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import com.google.protobuf.BlockingService;
@@ -377,4 +387,45 @@ public class AdminService extends Abstra
public String[] getGroupsForUser(String user) throws IOException {
return UserGroupInformation.createRemoteUser(user).getGroupNames();
}
+
+ @Override
+ public UpdateNodeResourceResponse updateNodeResource(
+ UpdateNodeResourceRequest request) throws YarnException, IOException {
+ Map<NodeId, ResourceOption> nodeResourceMap = request.getNodeResourceMap();
+ Set<NodeId> nodeIds = nodeResourceMap.keySet();
+ // verify nodes are all valid first.
+ // if any invalid nodes, throw exception instead of partially updating
+ // valid nodes.
+ for (NodeId nodeId : nodeIds) {
+ RMNode node = this.rmContext.getRMNodes().get(nodeId);
+ if (node == null) {
+ LOG.error("Resource update get failed on all nodes due to change "
+ + "resource on an unrecognized node: " + nodeId);
+ throw RPCUtil.getRemoteException(
+ "Resource update get failed on all nodes due to change resource "
+ + "on an unrecognized node: " + nodeId);
+ }
+ }
+
+ // do resource update on each node.
+ // Notice: it is still possible to have invalid NodeIDs as nodes decommission
+ // may happen just at the same time. This time, only log and skip absent
+ // nodes without throwing any exceptions.
+ for (Map.Entry<NodeId, ResourceOption> entry : nodeResourceMap.entrySet()) {
+ ResourceOption newResourceOption = entry.getValue();
+ NodeId nodeId = entry.getKey();
+ RMNode node = this.rmContext.getRMNodes().get(nodeId);
+ if (node == null) {
+ LOG.warn("Resource update get failed on an unrecognized node: " + nodeId);
+ } else {
+ node.setResourceOption(newResourceOption);
+ LOG.info("Update resource successfully on node(" + node.getNodeID()
+ +") with resource(" + newResourceOption.toString() + ")");
+ }
+ }
+ UpdateNodeResourceResponse response = recordFactory.newRecordInstance(
+ UpdateNodeResourceResponse.class);
+ return response;
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Fri Dec 20 01:01:18 2013
@@ -292,15 +292,15 @@ public class ApplicationMasterService ex
this.amLivelinessMonitor.receivedPing(applicationAttemptId);
- rmContext.getDispatcher().getEventHandler().handle(
- new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
- .getTrackingUrl(), request.getFinalApplicationStatus(), request
- .getDiagnostics()));
-
if (rmContext.getRMApps().get(applicationAttemptId.getApplicationId())
- .isAppSafeToUnregister()) {
+ .isAppSafeToTerminate()) {
return FinishApplicationMasterResponse.newInstance(true);
} else {
+ // keep sending the unregister event as RM may crash in the meanwhile.
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
+ .getTrackingUrl(), request.getFinalApplicationStatus(), request
+ .getDiagnostics()));
return FinishApplicationMasterResponse.newInstance(false);
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Dec 20 01:01:18 2013
@@ -380,14 +380,15 @@ public class ClientRMService extends Abs
+ ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
}
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMAppEvent(applicationId, RMAppEventType.KILL));
-
- RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
- AuditConstants.KILL_APP_REQUEST, "ClientRMService" , applicationId);
- KillApplicationResponse response = recordFactory
- .newRecordInstance(KillApplicationResponse.class);
- return response;
+ if (application.isAppSafeToTerminate()) {
+ RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
+ AuditConstants.KILL_APP_REQUEST, "ClientRMService", applicationId);
+ return KillApplicationResponse.newInstance(true);
+ } else {
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMAppEvent(applicationId, RMAppEventType.KILL));
+ return KillApplicationResponse.newInstance(false);
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Fri Dec 20 01:01:18 2013
@@ -287,11 +287,12 @@ public class FileSystemRMStateStore exte
}
@Override
- public synchronized void storeApplicationStateInternal(String appId,
+ public synchronized void storeApplicationStateInternal(ApplicationId appId,
ApplicationStateDataPBImpl appStateDataPB) throws Exception {
- Path appDirPath = getAppDir(rmAppRoot, appId);
+ String appIdStr = appId.toString();
+ Path appDirPath = getAppDir(rmAppRoot, appIdStr);
fs.mkdirs(appDirPath);
- Path nodeCreatePath = getNodePath(appDirPath, appId);
+ Path nodeCreatePath = getNodePath(appDirPath, appIdStr);
LOG.info("Storing info for app: " + appId + " at: " + nodeCreatePath);
byte[] appStateData = appStateDataPB.getProto().toByteArray();
@@ -306,10 +307,11 @@ public class FileSystemRMStateStore exte
}
@Override
- public synchronized void updateApplicationStateInternal(String appId,
+ public synchronized void updateApplicationStateInternal(ApplicationId appId,
ApplicationStateDataPBImpl appStateDataPB) throws Exception {
- Path appDirPath = getAppDir(rmAppRoot, appId);
- Path nodeCreatePath = getNodePath(appDirPath, appId);
+ String appIdStr = appId.toString();
+ Path appDirPath = getAppDir(rmAppRoot, appIdStr);
+ Path nodeCreatePath = getNodePath(appDirPath, appIdStr);
LOG.info("Updating info for app: " + appId + " at: " + nodeCreatePath);
byte[] appStateData = appStateDataPB.getProto().toByteArray();
@@ -325,14 +327,13 @@ public class FileSystemRMStateStore exte
@Override
public synchronized void storeApplicationAttemptStateInternal(
- String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateDataPB)
throws Exception {
- ApplicationAttemptId appAttemptId =
- ConverterUtils.toApplicationAttemptId(attemptId);
Path appDirPath =
getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
- Path nodeCreatePath = getNodePath(appDirPath, attemptId);
- LOG.info("Storing info for attempt: " + attemptId + " at: "
+ Path nodeCreatePath = getNodePath(appDirPath, appAttemptId.toString());
+ LOG.info("Storing info for attempt: " + appAttemptId + " at: "
+ nodeCreatePath);
byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
try {
@@ -340,21 +341,20 @@ public class FileSystemRMStateStore exte
// based on whether we have lost the right to write to FS
writeFile(nodeCreatePath, attemptStateData);
} catch (Exception e) {
- LOG.info("Error storing info for attempt: " + attemptId, e);
+ LOG.info("Error storing info for attempt: " + appAttemptId, e);
throw e;
}
}
@Override
public synchronized void updateApplicationAttemptStateInternal(
- String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateDataPB)
throws Exception {
- ApplicationAttemptId appAttemptId =
- ConverterUtils.toApplicationAttemptId(attemptId);
Path appDirPath =
getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
- Path nodeCreatePath = getNodePath(appDirPath, attemptId);
- LOG.info("Updating info for attempt: " + attemptId + " at: "
+ Path nodeCreatePath = getNodePath(appDirPath, appAttemptId.toString());
+ LOG.info("Updating info for attempt: " + appAttemptId + " at: "
+ nodeCreatePath);
byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
try {
@@ -362,7 +362,7 @@ public class FileSystemRMStateStore exte
// based on whether we have lost the right to write to FS
updateFile(nodeCreatePath, attemptStateData);
} catch (Exception e) {
- LOG.info("Error updating info for attempt: " + attemptId, e);
+ LOG.info("Error updating info for attempt: " + appAttemptId, e);
throw e;
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1552467&r1=1552466&r2=1552467&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Fri Dec 20 01:01:18 2013
@@ -80,7 +80,7 @@ public class MemoryRMStateStore extends
}
@Override
- public void storeApplicationStateInternal(String appId,
+ public void storeApplicationStateInternal(ApplicationId appId,
ApplicationStateDataPBImpl appStateData)
throws Exception {
ApplicationState appState =
@@ -88,11 +88,11 @@ public class MemoryRMStateStore extends
appStateData.getStartTime(),
appStateData.getApplicationSubmissionContext(),
appStateData.getUser());
- state.appState.put(appState.getAppId(), appState);
+ state.appState.put(appId, appState);
}
@Override
- public void updateApplicationStateInternal(String appId,
+ public void updateApplicationStateInternal(ApplicationId appId,
ApplicationStateDataPBImpl appStateData) throws Exception {
ApplicationState updatedAppState =
new ApplicationState(appStateData.getSubmitTime(),
@@ -102,21 +102,19 @@ public class MemoryRMStateStore extends
appStateData.getDiagnostics(), appStateData.getFinishTime());
LOG.info("Updating final state " + appStateData.getState() + " for app: "
+ appId);
- ApplicationId applicationId = updatedAppState.getAppId();
- if (state.appState.get(applicationId) != null) {
+ if (state.appState.get(appId) != null) {
// add the earlier attempts back
updatedAppState.attempts
- .putAll(state.appState.get(applicationId).attempts);
+ .putAll(state.appState.get(appId).attempts);
}
- state.appState.put(applicationId, updatedAppState);
+ state.appState.put(appId, updatedAppState);
}
@Override
- public synchronized void storeApplicationAttemptStateInternal(String attemptIdStr,
- ApplicationAttemptStateDataPBImpl attemptStateData)
- throws Exception {
- ApplicationAttemptId attemptId = ConverterUtils
- .toApplicationAttemptId(attemptIdStr);
+ public synchronized void storeApplicationAttemptStateInternal(
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateData)
+ throws Exception {
Credentials credentials = null;
if(attemptStateData.getAppAttemptTokens() != null){
DataInputByteBuffer dibb = new DataInputByteBuffer();
@@ -125,7 +123,7 @@ public class MemoryRMStateStore extends
credentials.readTokenStorageStream(dibb);
}
ApplicationAttemptState attemptState =
- new ApplicationAttemptState(attemptId,
+ new ApplicationAttemptState(appAttemptId,
attemptStateData.getMasterContainer(), credentials,
attemptStateData.getStartTime());
@@ -139,10 +137,9 @@ public class MemoryRMStateStore extends
@Override
public synchronized void updateApplicationAttemptStateInternal(
- String attemptIdStr, ApplicationAttemptStateDataPBImpl attemptStateData)
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateData)
throws Exception {
- ApplicationAttemptId attemptId =
- ConverterUtils.toApplicationAttemptId(attemptIdStr);
Credentials credentials = null;
if (attemptStateData.getAppAttemptTokens() != null) {
DataInputByteBuffer dibb = new DataInputByteBuffer();
@@ -151,7 +148,7 @@ public class MemoryRMStateStore extends
credentials.readTokenStorageStream(dibb);
}
ApplicationAttemptState updatedAttemptState =
- new ApplicationAttemptState(attemptId,
+ new ApplicationAttemptState(appAttemptId,
attemptStateData.getMasterContainer(), credentials,
attemptStateData.getStartTime(), attemptStateData.getState(),
attemptStateData.getFinalTrackingUrl(),