You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/05 04:43:33 UTC
svn commit: r1464815 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ap...
Author: szetszwo
Date: Fri Apr 5 02:43:29 2013
New Revision: 1464815
URL: http://svn.apache.org/r1464815
Log:
Merge r1462698 through r1464807 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/
- copied from r1464807, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
- copied unchanged from r1464807, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java
- copied unchanged from r1464807, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalCacheDirectoryManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
- copied unchanged from r1464807, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Fri Apr 5 02:43:29 2013
@@ -54,6 +54,9 @@ Trunk - Unreleased
YARN-359. Fixing commands for container signalling in Windows. (Chris Nauroth
via vinodkv)
+
+ YARN-524 TestYarnVersionInfo failing if generated properties doesn't
+ include an SVN URL(stevel)
Release 2.0.5-beta - UNRELEASED
@@ -66,6 +69,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-440. Flatten RegisterNodeManagerResponse. (Xuan Gong via sseth)
+ YARN-536. Removed the unused objects ContainerStatus and ContainerStatus from
+ Container which also don't belong to the container. (Xuan Gong via vinodkv)
+
NEW FEATURES
IMPROVEMENTS
@@ -103,6 +109,20 @@ Release 2.0.5-beta - UNRELEASED
YARN-450. Define value for * in the scheduling protocol (Zhijie Shen via
bikas)
+ YARN-475. Remove a unused constant in the public API -
+ ApplicationConstants.AM_APP_ATTEMPT_ID_ENV. (Hitesh Shah via vinodkv)
+
+ YARN-309. Changed NodeManager to obtain heart-beat interval from the
+ ResourceManager. (Xuan Gong via vinodkv)
+
+ YARN-447. Move ApplicationComparator in CapacityScheduler to use comparator
+ in ApplicationId. (Nemon Lou via vinodkv)
+
+ YARN-381. Improve fair scheduler docs. (Sandy Ryza via tomwhite)
+
+ YARN-458. YARN daemon addresses must be placed in many different configs.
+ (sandyr via tucu)
+
OPTIMIZATIONS
BUG FIXES
@@ -134,9 +154,6 @@ Release 2.0.5-beta - UNRELEASED
YARN-485. TestProcfsProcessTree#testProcessTree() doesn't wait long enough
for the process to die. (kkambatl via tucu)
- YARN-470. Support a way to disable resource monitoring on the NodeManager.
- (Siddharth Seth via hitesh)
-
YARN-71. Fix the NodeManager to clean up local-dirs on restart.
(Xuan Gong via sseth)
@@ -161,6 +178,23 @@ Release 2.0.5-beta - UNRELEASED
YARN-515. Node Manager not getting the master key. (Robert Joseph Evans
via jlowe)
+ YARN-382. SchedulerUtils improve way normalizeRequest sets the resource
+ capabilities. (Zhijie Shen via bikas)
+
+ YARN-467. Modify public distributed cache to localize files such that no
+ local directory hits unix file count limits and thus prevent job failures.
+ (Omkar Vinit Joshi via vinodkv)
+
+ YARN-101. Fix NodeManager heartbeat processing to not lose track of completed
+ containers in case of dropped heartbeats. (Xuan Gong via vinodkv)
+
+ YARN-538. RM address DNS lookup can cause unnecessary slowness on every JHS
+ page load. (sandyr via tucu)
+
+ YARN-532. Change RMAdmin and Localization client protocol PB implementations
+ to implement closeable so that they can be stopped when needed via
+ RPC.stopProxy(). (Siddharth Seth via vinodkv)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
@@ -176,6 +210,9 @@ Release 2.0.4-alpha - UNRELEASED
YARN-429. capacity-scheduler config missing from yarn-test artifact.
(sseth via hitesh)
+ YARN-470. Support a way to disable resource monitoring on the NodeManager.
+ (Siddharth Seth via hitesh)
+
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Fri Apr 5 02:43:29 2013
@@ -256,4 +256,18 @@
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Null pointer exception needs to be ignored here as this is never going to occur. -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourcesTrackerImpl" />
+ <Method name="decrementFileCountForLocalCacheDirectory" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
+
+ <!-- Null pointer exception needs to be ignored here as this is never going to occur. -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourcesTrackerImpl" />
+ <Method name="getPathForLocalization" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
+
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Fri Apr 5 02:43:29 2013
@@ -42,12 +42,6 @@ public interface ApplicationConstants {
* only
*/
public static final String AM_CONTAINER_ID_ENV = "AM_CONTAINER_ID";
-
- /**
- * The environment variable for APPLICATION_ATTEMPT_ID. Set in AppMaster
- * environment only
- */
- public static final String AM_APP_ATTEMPT_ID_ENV = "AM_APP_ATTEMPT_ID";
/**
* The environment variable for the NM_HOST. Set in the AppMaster environment
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java Fri Apr 5 02:43:29 2013
@@ -125,18 +125,6 @@ public interface Container extends Compa
void setPriority(Priority priority);
/**
- * Get the current <code>ContainerState</code> of the container.
- * @return current <code>ContainerState</code> of the container
- */
- @Public
- @Stable
- ContainerState getState();
-
- @Private
- @Unstable
- void setState(ContainerState state);
-
- /**
* Get the <code>ContainerToken</code> for the container.
* @return <code>ContainerToken</code> for the container
*/
@@ -147,16 +135,4 @@ public interface Container extends Compa
@Private
@Unstable
void setContainerToken(ContainerToken containerToken);
-
- /**
- * Get the <code>ContainerStatus</code> of the container.
- * @return <code>ContainerStatus</code> of the container
- */
- @Public
- @Stable
- ContainerStatus getContainerStatus();
-
- @Private
- @Unstable
- void setContainerStatus(ContainerStatus containerStatus);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java Fri Apr 5 02:43:29 2013
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.api.recor
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -31,12 +29,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-import org.apache.hadoop.yarn.util.ProtoUtils;
public class ContainerPBImpl extends ProtoBase<ContainerProto> implements Container {
@@ -49,7 +44,6 @@ public class ContainerPBImpl extends Pro
private Resource resource = null;
private Priority priority = null;
private ContainerToken containerToken = null;
- private ContainerStatus containerStatus = null;
public ContainerPBImpl() {
builder = ContainerProto.newBuilder();
@@ -94,11 +88,6 @@ public class ContainerPBImpl extends Pro
builder.getContainerToken())) {
builder.setContainerToken(convertToProtoFormat(this.containerToken));
}
- if (this.containerStatus != null
- && !((ContainerStatusPBImpl) this.containerStatus).getProto().equals(
- builder.getContainerStatus())) {
- builder.setContainerStatus(convertToProtoFormat(this.containerStatus));
- }
}
private void mergeLocalToProto() {
@@ -115,27 +104,8 @@ public class ContainerPBImpl extends Pro
}
viaProto = false;
}
-
-
- @Override
- public ContainerState getState() {
- ContainerProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasState()) {
- return null;
- }
- return convertFromProtoFormat(p.getState());
- }
@Override
- public void setState(ContainerState state) {
- maybeInitBuilder();
- if (state == null) {
- builder.clearState();
- return;
- }
- builder.setState(convertToProtoFormat(state));
- }
- @Override
public ContainerId getId() {
ContainerProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerId != null) {
@@ -260,35 +230,6 @@ public class ContainerPBImpl extends Pro
this.containerToken = containerToken;
}
- @Override
- public ContainerStatus getContainerStatus() {
- ContainerProtoOrBuilder p = viaProto ? proto : builder;
- if (this.containerStatus != null) {
- return this.containerStatus;
- }
- if (!p.hasContainerStatus()) {
- return null;
- }
- this.containerStatus = convertFromProtoFormat(p.getContainerStatus());
- return this.containerStatus;
- }
-
- @Override
- public void setContainerStatus(ContainerStatus containerStatus) {
- maybeInitBuilder();
- if (containerStatus == null)
- builder.clearContainerStatus();
- this.containerStatus = containerStatus;
- }
-
- private ContainerStateProto convertToProtoFormat(ContainerState e) {
- return ProtoUtils.convertToProtoFormat(e);
- }
-
- private ContainerState convertFromProtoFormat(ContainerStateProto e) {
- return ProtoUtils.convertFromProtoFormat(e);
- }
-
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
return new ContainerIdPBImpl(p);
}
@@ -329,14 +270,6 @@ public class ContainerPBImpl extends Pro
return ((ContainerTokenPBImpl)t).getProto();
}
- private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto p) {
- return new ContainerStatusPBImpl(p);
- }
-
- private ContainerStatusProto convertToProtoFormat(ContainerStatus t) {
- return ((ContainerStatusPBImpl)t).getProto();
- }
-
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("Container: [");
@@ -345,9 +278,7 @@ public class ContainerPBImpl extends Pro
sb.append("NodeHttpAddress: ").append(getNodeHttpAddress()).append(", ");
sb.append("Resource: ").append(getResource()).append(", ");
sb.append("Priority: ").append(getPriority()).append(", ");
- sb.append("State: ").append(getState()).append(", ");
sb.append("Token: ").append(getContainerToken()).append(", ");
- sb.append("Status: ").append(getContainerStatus());
sb.append("]");
return sb.toString();
}
@@ -357,16 +288,7 @@ public class ContainerPBImpl extends Pro
public int compareTo(Container other) {
if (this.getId().compareTo(other.getId()) == 0) {
if (this.getNodeId().compareTo(other.getNodeId()) == 0) {
- if (this.getResource().compareTo(other.getResource()) == 0) {
- if (this.getState().compareTo(other.getState()) == 0) {
- //ContainerToken
- return this.getState().compareTo(other.getState());
- } else {
- return this.getState().compareTo(other.getState());
- }
- } else {
- return this.getResource().compareTo(other.getResource());
- }
+ return this.getResource().compareTo(other.getResource());
} else {
return this.getNodeId().compareTo(other.getNodeId());
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Apr 5 02:43:29 2013
@@ -67,9 +67,7 @@ message ContainerProto {
optional string node_http_address = 3;
optional ResourceProto resource = 4;
optional PriorityProto priority = 5;
- optional ContainerStateProto state = 6;
- optional hadoop.common.TokenProto container_token = 7;
- optional ContainerStatusProto container_status = 8;
+ optional hadoop.common.TokenProto container_token = 6;
}
enum YarnApplicationStateProto {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Apr 5 02:43:29 2013
@@ -608,7 +608,6 @@ public class ApplicationMaster {
+ ", containerNode=" + allocatedContainer.getNodeId().getHost()
+ ":" + allocatedContainer.getNodeId().getPort()
+ ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress()
- + ", containerState" + allocatedContainer.getState()
+ ", containerResourceMemory"
+ allocatedContainer.getResource().getMemory());
// + ", containerToken"
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java Fri Apr 5 02:43:29 2013
@@ -57,10 +57,11 @@ import org.apache.hadoop.yarn.util.Recor
* unmanagedAM is an AM that is not launched and managed by the RM. The client
* creates a new application on the RM and negotiates a new attempt id. Then it
* waits for the RM app state to reach be YarnApplicationState.ACCEPTED after
- * which it spawns the AM in another process and passes it the attempt id via
- * env variable ApplicationConstants.AM_APP_ATTEMPT_ID_ENV. The AM can be in any
- * language. The AM can register with the RM using the attempt id and proceed as
- * normal. The client redirects app stdout and stderr to its own stdout and
+ * which it spawns the AM in another process and passes it the container id via
+ * env variable ApplicationConstants.AM_CONTAINER_ID_ENV. The AM can be in any
+ * language. The AM can register with the RM using the attempt id obtained
+ * from the container id and proceed as normal.
+ * The client redirects app stdout and stderr to its own stdout and
* stderr and waits for the AM process to exit. Then it waits for the RM to
* report app completion.
*/
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/RMAdminProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/RMAdminProtocolPBClientImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/RMAdminProtocolPBClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/RMAdminProtocolPBClientImpl.java Fri Apr 5 02:43:29 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.impl.pb.client;
+import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -65,7 +66,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import com.google.protobuf.ServiceException;
-public class RMAdminProtocolPBClientImpl implements RMAdminProtocol {
+public class RMAdminProtocolPBClientImpl implements RMAdminProtocol, Closeable {
private RMAdminProtocolPB proxy;
@@ -78,6 +79,13 @@ public class RMAdminProtocolPBClientImpl
}
@Override
+ public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+
+ @Override
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
throws YarnRemoteException {
RefreshQueuesRequestProto requestProto =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Apr 5 02:43:29 2013
@@ -219,6 +219,11 @@ public class YarnConfiguration extends C
public static final String DEFAULT_RM_SCHEDULER =
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
+ /** RM set next Heartbeat interval for NM */
+ public static final String RM_NM_HEARTBEAT_INTERVAL_MS =
+ RM_PREFIX + "nodemanagers.heartbeat-interval-ms";
+ public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000;
+
//Delegation token related keys
public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";
@@ -329,19 +334,21 @@ public class YarnConfiguration extends C
NM_PREFIX + "delete.thread-count";
public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4;
- // TODO: Should this instead be dictated by RM?
- /** Heartbeat interval to RM*/
- public static final String NM_TO_RM_HEARTBEAT_INTERVAL_MS =
- NM_PREFIX + "heartbeat.interval-ms";
- public static final int DEFAULT_NM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000;
-
/** Keytab for NM.*/
public static final String NM_KEYTAB = NM_PREFIX + "keytab";
/**List of directories to store localized files in.*/
public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs";
public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir";
-
+
+ /**
+ * Number of files in each localized directories
+ * Avoid tuning this too low.
+ */
+ public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY =
+ NM_PREFIX + "local-cache.max-files-per-directory";
+ public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192;
+
/** Address where the localizer IPC is.*/
public static final String NM_LOCALIZER_ADDRESS =
NM_PREFIX + "localizer.address";
@@ -710,21 +717,16 @@ public class YarnConfiguration extends C
}
public static String getRMWebAppHostAndPort(Configuration conf) {
- int port = conf.getSocketAddr(
+ InetSocketAddress address = conf.getSocketAddr(
YarnConfiguration.RM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_PORT).getPort();
- // Use apps manager address to figure out the host for webapp
- String host = conf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT).getHostName();
- InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port);
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ address = NetUtils.getConnectAddress(address);
StringBuffer sb = new StringBuffer();
InetAddress resolved = address.getAddress();
if (resolved == null || resolved.isAnyLocalAddress() ||
resolved.isLoopbackAddress()) {
- String lh = host;
+ String lh = address.getHostName();
try {
lh = InetAddress.getLocalHost().getCanonicalHostName();
} catch (UnknownHostException e) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Fri Apr 5 02:43:29 2013
@@ -246,11 +246,6 @@ public class BuilderUtils {
container.setNodeHttpAddress(nodeHttpAddress);
container.setResource(resource);
container.setPriority(priority);
- container.setState(ContainerState.NEW);
- ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class);
- containerStatus.setContainerId(containerId);
- containerStatus.setState(ContainerState.NEW);
- container.setContainerStatus(containerStatus);
container.setContainerToken(containerToken);
return container;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Apr 5 02:43:29 2013
@@ -59,9 +59,15 @@
<!-- Resource Manager Configs -->
<property>
+ <description>The hostname of the RM.</description>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
- <value>0.0.0.0:8032</value>
+ <value>${yarn.resourcemanager.hostname}:8032</value>
</property>
<property>
@@ -84,7 +90,7 @@
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>0.0.0.0:8030</value>
+ <value>${yarn.resourcemanager.hostname}:8030</value>
</property>
<property>
@@ -96,12 +102,12 @@
<property>
<description>The address of the RM web application.</description>
<name>yarn.resourcemanager.webapp.address</name>
- <value>0.0.0.0:8088</value>
+ <value>${yarn.resourcemanager.hostname}:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
- <value>0.0.0.0:8031</value>
+ <value>${yarn.resourcemanager.hostname}:8031</value>
</property>
<property>
@@ -119,7 +125,7 @@
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
- <value>0.0.0.0:8033</value>
+ <value>${yarn.resourcemanager.hostname}:8033</value>
</property>
<property>
@@ -278,11 +284,23 @@
<value>86400</value>
</property>
+ <property>
+ <description>The heart-beat interval in milliseconds for every NodeManager in the cluster.</description>
+ <name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
+ <value>1000</value>
+ </property>
+
<!-- Node Manager Configs -->
<property>
+ <description>The hostname of the NM.</description>
+ <name>yarn.nodemanager.hostname</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
<description>The address of the container manager in the NM.</description>
<name>yarn.nodemanager.address</name>
- <value>0.0.0.0:0</value>
+ <value>${yarn.nodemanager.hostname}:0</value>
</property>
<property>
@@ -337,12 +355,6 @@
</property>
<property>
- <description>Heartbeat interval to RM</description>
- <name>yarn.nodemanager.heartbeat.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
<description>Keytab for NM.</description>
<name>yarn.nodemanager.keytab</name>
<value>/etc/krb5.keytab</value>
@@ -360,9 +372,28 @@
</property>
<property>
+ <description>It limits the maximum number of files which will be localized
+ in a single local directory. If the limit is reached then sub-directories
+ will be created and new files will be localized in them. If it is set to
+ a value less than or equal to 36 [which are sub-directories (0-9 and then
+ a-z)] then NodeManager will fail to start. For example; [for public
+ cache] if this is configured with a value of 40 ( 4 files +
+ 36 sub-directories) and the local-dir is "/tmp/local-dir1" then it will
+ allow 4 files to be created directly inside "/tmp/local-dir1/filecache".
+ For files that are localized further it will create a sub-directory "0"
+ inside "/tmp/local-dir1/filecache" and will localize files inside it
+ until it becomes full. If a file is removed from a sub-directory that
+ is marked full, then that sub-directory will be used back again to
+ localize files.
+ </description>
+ <name>yarn.nodemanager.local-cache.max-files-per-directory</name>
+ <value>8192</value>
+ </property>
+
+ <property>
<description>Address where the localizer IPC is.</description>
<name>yarn.nodemanager.localizer.address</name>
- <value>0.0.0.0:8040</value>
+ <value>${yarn.nodemanager.hostname}:8040</value>
</property>
<property>
@@ -493,7 +524,7 @@
<property>
<description>NM Webapp address.</description>
<name>yarn.nodemanager.webapp.address</name>
- <value>0.0.0.0:8042</value>
+ <value>${yarn.nodemanager.hostname}:8042</value>
</property>
<property>
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java Fri Apr 5 02:43:29 2013
@@ -40,11 +40,11 @@ public class TestYarnVersionInfo extends
// make sure they aren't Unknown
assertTrue("getVersion returned Unknown", !YarnVersionInfo.getVersion().equals("Unknown"));
assertTrue("getUser returned Unknown", !YarnVersionInfo.getUser().equals("Unknown"));
- assertTrue("getUrl returned Unknown", !YarnVersionInfo.getUrl().equals("Unknown"));
assertTrue("getSrcChecksum returned Unknown", !YarnVersionInfo.getSrcChecksum().equals("Unknown"));
// these could be Unknown if the VersionInfo generated from code not in svn or git
// so just check that they return something
+ assertNotNull("getUrl returned null", YarnVersionInfo.getUrl());
assertNotNull("getRevision returned null", YarnVersionInfo.getRevision());
assertNotNull("getBranch returned null", YarnVersionInfo.getBranch());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java Fri Apr 5 02:43:29 2013
@@ -42,4 +42,7 @@ public interface NodeHeartbeatResponse {
void addAllContainersToCleanup(List<ContainerId> containers);
void addAllApplicationsToCleanup(List<ApplicationId> applications);
+
+ long getNextHeartBeatInterval();
+ void setNextHeartBeatInterval(long nextHeartBeatInterval);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java Fri Apr 5 02:43:29 2013
@@ -271,6 +271,18 @@ public class NodeHeartbeatResponsePBImpl
builder.addAllApplicationsToCleanup(iterable);
}
+ @Override
+ public long getNextHeartBeatInterval() {
+ NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getNextHeartBeatInterval());
+ }
+
+ @Override
+ public void setNextHeartBeatInterval(long nextHeartBeatInterval) {
+ maybeInitBuilder();
+ builder.setNextHeartBeatInterval(nextHeartBeatInterval);
+ }
+
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
return new ContainerIdPBImpl(p);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto Fri Apr 5 02:43:29 2013
@@ -47,4 +47,5 @@ message NodeHeartbeatResponseProto {
optional NodeActionProto nodeAction = 3;
repeated ContainerIdProto containers_to_cleanup = 4;
repeated ApplicationIdProto applications_to_cleanup = 5;
+ optional int64 nextHeartBeatInterval = 6;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Apr 5 02:43:29 2013
@@ -119,6 +119,10 @@ public class NodeManager extends Composi
return new DeletionService(exec);
}
+ protected NMContext createNMContext(NMContainerTokenSecretManager containerTokenSecretManager) {
+ return new NMContext(containerTokenSecretManager);
+ }
+
protected void doSecureLogin() throws IOException {
SecurityUtil.login(getConfig(), YarnConfiguration.NM_KEYTAB,
YarnConfiguration.NM_PRINCIPAL);
@@ -137,7 +141,7 @@ public class NodeManager extends Composi
containerTokenSecretManager = new NMContainerTokenSecretManager(conf);
}
- this.context = new NMContext(containerTokenSecretManager);
+ this.context = createNMContext(containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Apr 5 02:43:29 2013
@@ -71,7 +71,7 @@ public class NodeStatusUpdaterImpl exten
private final Dispatcher dispatcher;
private NodeId nodeId;
- private long heartBeatInterval;
+ private long nextHeartBeatInterval;
private ResourceTracker resourceTracker;
private InetSocketAddress rmAddress;
private Resource totalResource;
@@ -88,6 +88,10 @@ public class NodeStatusUpdaterImpl exten
private final NodeHealthCheckerService healthChecker;
private final NodeManagerMetrics metrics;
+ private boolean previousHeartBeatSucceeded;
+ private List<ContainerStatus> previousContainersStatuses =
+ new ArrayList<ContainerStatus>();
+
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
super(NodeStatusUpdaterImpl.class.getName());
@@ -95,6 +99,7 @@ public class NodeStatusUpdaterImpl exten
this.context = context;
this.dispatcher = dispatcher;
this.metrics = metrics;
+ this.previousHeartBeatSucceeded = true;
}
@Override
@@ -103,9 +108,7 @@ public class NodeStatusUpdaterImpl exten
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- this.heartBeatInterval =
- conf.getLong(YarnConfiguration.NM_TO_RM_HEARTBEAT_INTERVAL_MS,
- YarnConfiguration.DEFAULT_NM_TO_RM_HEARTBEAT_INTERVAL_MS);
+
int memoryMb =
conf.getInt(
YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
@@ -316,8 +319,14 @@ public class NodeStatusUpdaterImpl exten
NodeStatus nodeStatus = recordFactory.newRecordInstance(NodeStatus.class);
nodeStatus.setNodeId(this.nodeId);
- int numActiveContainers = 0;
List<ContainerStatus> containersStatuses = new ArrayList<ContainerStatus>();
+ if(previousHeartBeatSucceeded) {
+ previousContainersStatuses.clear();
+ } else {
+ containersStatuses.addAll(previousContainersStatuses);
+ }
+
+ int numActiveContainers = 0;
for (Iterator<Entry<ContainerId, Container>> i =
this.context.getContainers().entrySet().iterator(); i.hasNext();) {
Entry<ContainerId, Container> e = i.next();
@@ -332,6 +341,7 @@ public class NodeStatusUpdaterImpl exten
LOG.info("Sending out status for container: " + containerStatus);
if (containerStatus.getState() == ContainerState.COMPLETE) {
+ previousContainersStatuses.add(containerStatus);
// Remove
i.remove();
@@ -394,9 +404,6 @@ public class NodeStatusUpdaterImpl exten
while (!isStopped) {
// Send heartbeat
try {
- synchronized (heartbeatMonitor) {
- heartbeatMonitor.wait(heartBeatInterval);
- }
NodeStatus nodeStatus = getNodeStatus();
nodeStatus.setResponseId(lastHeartBeatID);
@@ -409,7 +416,9 @@ public class NodeStatusUpdaterImpl exten
}
NodeHeartbeatResponse response =
resourceTracker.nodeHeartbeat(request);
-
+ previousHeartBeatSucceeded = true;
+ //get next heartbeat interval from response
+ nextHeartBeatInterval = response.getNextHeartBeatInterval();
// See if the master-key has rolled over
if (isSecurityEnabled()) {
MasterKey updatedMasterKey = response.getMasterKey();
@@ -453,9 +462,21 @@ public class NodeStatusUpdaterImpl exten
new CMgrCompletedAppsEvent(appsToCleanup));
}
} catch (Throwable e) {
+ previousHeartBeatSucceeded = false;
// TODO Better error handling. Thread can die with the rest of the
// NM still running.
LOG.error("Caught exception in status-updater", e);
+ } finally {
+ synchronized (heartbeatMonitor) {
+ nextHeartBeatInterval = nextHeartBeatInterval <= 0 ?
+ YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS :
+ nextHeartBeatInterval;
+ try {
+ heartbeatMonitor.wait(nextHeartBeatInterval);
+ } catch (InterruptedException e) {
+ // Do Nothing
+ }
+ }
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java Fri Apr 5 02:43:29 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb.client;
+import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -35,7 +36,8 @@ import org.apache.hadoop.yarn.server.nod
import com.google.protobuf.ServiceException;
-public class LocalizationProtocolPBClientImpl implements LocalizationProtocol {
+public class LocalizationProtocolPBClientImpl implements LocalizationProtocol,
+ Closeable {
private LocalizationProtocolPB proxy;
@@ -44,7 +46,14 @@ public class LocalizationProtocolPBClien
proxy = (LocalizationProtocolPB)RPC.getProxy(
LocalizationProtocolPB.class, clientVersion, addr, conf);
}
-
+
+ @Override
+ public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+
@Override
public LocalizerHeartbeatResponse heartbeat(LocalizerStatus status)
throws YarnRemoteException {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java Fri Apr 5 02:43:29 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -35,6 +36,11 @@ interface LocalResourcesTracker
boolean remove(LocalizedResource req, DeletionService delService);
+ Path getPathForLocalization(LocalResourceRequest req, Path localDirPath);
+
String getUser();
+ // TODO: Remove this in favour of EventHandler.handle
+ void localizationCompleted(LocalResourceRequest req, boolean success);
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java Fri Apr 5 02:43:29 2013
@@ -26,12 +26,13 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEventType;
+
/**
* A collection of {@link LocalizedResource}s all of same
@@ -49,17 +50,43 @@ class LocalResourcesTrackerImpl implemen
private final String user;
private final Dispatcher dispatcher;
private final ConcurrentMap<LocalResourceRequest,LocalizedResource> localrsrc;
+ private Configuration conf;
+ /*
+ * This flag controls whether this resource tracker uses hierarchical
+ * directories or not. For PRIVATE and PUBLIC resource trackers it
+ * will be set whereas for APPLICATION resource tracker it would
+ * be false.
+ */
+ private final boolean useLocalCacheDirectoryManager;
+ private ConcurrentHashMap<Path, LocalCacheDirectoryManager> directoryManagers;
+ /*
+ * It is used to keep track of resource into hierarchical directory
+ * while it is getting downloaded. It is useful for reference counting
+ * in case resource localization fails.
+ */
+ private ConcurrentHashMap<LocalResourceRequest, Path>
+ inProgressLocalResourcesMap;
- public LocalResourcesTrackerImpl(String user, Dispatcher dispatcher) {
+ public LocalResourcesTrackerImpl(String user, Dispatcher dispatcher,
+ boolean useLocalCacheDirectoryManager, Configuration conf) {
this(user, dispatcher,
- new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>());
+ new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(),
+ useLocalCacheDirectoryManager, conf);
}
LocalResourcesTrackerImpl(String user, Dispatcher dispatcher,
- ConcurrentMap<LocalResourceRequest,LocalizedResource> localrsrc) {
+ ConcurrentMap<LocalResourceRequest,LocalizedResource> localrsrc,
+ boolean useLocalCacheDirectoryManager, Configuration conf) {
this.user = user;
this.dispatcher = dispatcher;
this.localrsrc = localrsrc;
+ this.useLocalCacheDirectoryManager = useLocalCacheDirectoryManager;
+ if ( this.useLocalCacheDirectoryManager) {
+ directoryManagers = new ConcurrentHashMap<Path, LocalCacheDirectoryManager>();
+ inProgressLocalResourcesMap =
+ new ConcurrentHashMap<LocalResourceRequest, Path>();
+ }
+ this.conf = conf;
}
@Override
@@ -73,6 +100,7 @@ class LocalResourcesTrackerImpl implemen
LOG.info("Resource " + rsrc.getLocalPath()
+ " is missing, localizing it again");
localrsrc.remove(req);
+ decrementFileCountForLocalCacheDirectory(req, rsrc);
rsrc = null;
}
if (null == rsrc) {
@@ -90,7 +118,52 @@ class LocalResourcesTrackerImpl implemen
rsrc.handle(event);
}
- /**
+ /*
+ * Update the file-count statistics for a local cache-directory.
+ * This will retrieve the localized path for the resource from
+ * 1) inProgressRsrcMap if the resource was under localization and it
+ * failed.
+ * 2) LocalizedResource if the resource is already localized.
+ * From this path it will identify the local directory under which the
+ * resource was localized. Then rest of the path will be used to decrement
+ * file count for the HierarchicalSubDirectory pointing to this relative
+ * path.
+ */
+ private void decrementFileCountForLocalCacheDirectory(LocalResourceRequest req,
+ LocalizedResource rsrc) {
+ if ( useLocalCacheDirectoryManager) {
+ Path rsrcPath = null;
+ if (inProgressLocalResourcesMap.containsKey(req)) {
+ // This happens when localization of a resource fails.
+ rsrcPath = inProgressLocalResourcesMap.remove(req);
+ } else if (rsrc != null && rsrc.getLocalPath() != null) {
+ rsrcPath = rsrc.getLocalPath().getParent().getParent();
+ }
+ if (rsrcPath != null) {
+ Path parentPath = new Path(rsrcPath.toUri().getRawPath());
+ while (!directoryManagers.containsKey(parentPath)) {
+ parentPath = parentPath.getParent();
+ if ( parentPath == null) {
+ return;
+ }
+ }
+ if ( parentPath != null) {
+ String parentDir = parentPath.toUri().getRawPath().toString();
+ LocalCacheDirectoryManager dir = directoryManagers.get(parentPath);
+ String rsrcDir = rsrcPath.toUri().getRawPath();
+ if (rsrcDir.equals(parentDir)) {
+ dir.decrementFileCountForPath("");
+ } else {
+ dir.decrementFileCountForPath(
+ rsrcDir.substring(
+ parentDir.length() + 1));
+ }
+ }
+ }
+ }
+ }
+
+/**
* This module checks if the resource which was localized is already present
* or not
*
@@ -100,7 +173,8 @@ class LocalResourcesTrackerImpl implemen
public boolean isResourcePresent(LocalizedResource rsrc) {
boolean ret = true;
if (rsrc.getState() == ResourceState.LOCALIZED) {
- File file = new File(rsrc.getLocalPath().toUri().getRawPath().toString());
+ File file = new File(rsrc.getLocalPath().toUri().getRawPath().
+ toString());
if (!file.exists()) {
ret = false;
}
@@ -133,11 +207,11 @@ class LocalResourcesTrackerImpl implemen
if (ResourceState.LOCALIZED.equals(rsrc.getState())) {
delService.delete(getUser(), getPathToDelete(rsrc.getLocalPath()));
}
+ decrementFileCountForLocalCacheDirectory(rem.getRequest(), rsrc);
return true;
}
}
-
/**
* Returns the path up to the random directory component.
*/
@@ -163,4 +237,50 @@ class LocalResourcesTrackerImpl implemen
public Iterator<LocalizedResource> iterator() {
return localrsrc.values().iterator();
}
-}
+
+ /**
+ * @return {@link Path} absolute path for localization which includes local
+ * directory path and the relative hierarchical path (if use local
+ * cache directory manager is enabled)
+ *
+ * @param {@link LocalResourceRequest} Resource localization request to
+ * localize the resource.
+ * @param {@link Path} local directory path
+ */
+ @Override
+ public Path
+ getPathForLocalization(LocalResourceRequest req, Path localDirPath) {
+ if (useLocalCacheDirectoryManager && localDirPath != null) {
+
+ if (!directoryManagers.containsKey(localDirPath)) {
+ directoryManagers.putIfAbsent(localDirPath,
+ new LocalCacheDirectoryManager(conf));
+ }
+ LocalCacheDirectoryManager dir = directoryManagers.get(localDirPath);
+
+ Path rPath = localDirPath;
+ String hierarchicalPath = dir.getRelativePathForLocalization();
+ // For most of the scenarios we will get root path only which
+ // is an empty string
+ if (!hierarchicalPath.isEmpty()) {
+ rPath = new Path(localDirPath, hierarchicalPath);
+ }
+ inProgressLocalResourcesMap.put(req, rPath);
+ return rPath;
+ } else {
+ return localDirPath;
+ }
+ }
+
+ @Override
+ public void localizationCompleted(LocalResourceRequest req,
+ boolean success) {
+ if (useLocalCacheDirectoryManager) {
+ if (!success) {
+ decrementFileCountForLocalCacheDirectory(req, null);
+ } else {
+ inProgressLocalResourcesMap.remove(req);
+ }
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Fri Apr 5 02:43:29 2013
@@ -64,6 +64,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -130,7 +131,7 @@ public class ResourceLocalizationService
private RecordFactory recordFactory;
private final ScheduledExecutorService cacheCleanup;
- private final LocalResourcesTracker publicRsrc;
+ private LocalResourcesTracker publicRsrc;
private LocalDirsHandlerService dirsHandler;
@@ -158,7 +159,6 @@ public class ResourceLocalizationService
this.delService = delService;
this.dirsHandler = dirsHandler;
- this.publicRsrc = new LocalResourcesTrackerImpl(null, dispatcher);
this.cacheCleanup = new ScheduledThreadPoolExecutor(1,
new ThreadFactoryBuilder()
.setNameFormat("ResourceLocalizationService Cache Cleanup")
@@ -173,8 +173,26 @@ public class ResourceLocalizationService
}
}
+ private void validateConf(Configuration conf) {
+ int perDirFileLimit =
+ conf.getInt(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY,
+ YarnConfiguration.DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY);
+ if (perDirFileLimit <= 36) {
+ LOG.error(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
+ + " parameter is configured with very low value.");
+ throw new YarnException(
+ YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
+ + " parameter is configured with a value less than 37.");
+ } else {
+ LOG.info("per directory file limit = " + perDirFileLimit);
+ }
+ }
+
@Override
public void init(Configuration conf) {
+ this.validateConf(conf);
+ this.publicRsrc =
+ new LocalResourcesTrackerImpl(null, dispatcher, true, conf);
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
try {
@@ -212,6 +230,7 @@ public class ResourceLocalizationService
YarnConfiguration.NM_LOCALIZER_ADDRESS,
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
+
localizerTracker = createLocalizerTracker(conf);
addService(localizerTracker);
dispatcher.register(LocalizerEventType.class, localizerTracker);
@@ -306,15 +325,17 @@ public class ResourceLocalizationService
private void handleInitApplicationResources(Application app) {
// 0) Create application tracking structs
String userName = app.getUser();
- privateRsrc.putIfAbsent(userName,
- new LocalResourcesTrackerImpl(userName, dispatcher));
- if (null != appRsrc.putIfAbsent(ConverterUtils.toString(app.getAppId()),
- new LocalResourcesTrackerImpl(app.getUser(), dispatcher))) {
+ privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName,
+ dispatcher, false, super.getConfig()));
+ if (null != appRsrc.putIfAbsent(
+ ConverterUtils.toString(app.getAppId()),
+ new LocalResourcesTrackerImpl(app.getUser(), dispatcher, false, super
+ .getConfig()))) {
LOG.warn("Initializing application " + app + " already present");
assert false; // TODO: FIXME assert doesn't help
// ^ The condition is benign. Tests should fail and it
- // should appear in logs, but it's an internal error
- // that should have no effect on applications
+ // should appear in logs, but it's an internal error
+ // that should have no effect on applications
}
// 1) Signal container init
//
@@ -620,6 +641,13 @@ public class ResourceLocalizationService
Path publicDirDestPath = dirsHandler.getLocalPathForWrite(
"." + Path.SEPARATOR + ContainerLocalizer.FILECACHE,
ContainerLocalizer.getEstimatedSize(resource), true);
+ Path hierarchicalPath =
+ publicRsrc.getPathForLocalization(key, publicDirDestPath);
+ if (!hierarchicalPath.equals(publicDirDestPath)) {
+ publicDirDestPath = hierarchicalPath;
+ DiskChecker.checkDir(
+ new File(publicDirDestPath.toUri().getPath()));
+ }
pending.put(queue.submit(new FSDownload(
lfs, null, conf, publicDirDestPath, resource, new Random())),
request);
@@ -654,19 +682,21 @@ public class ResourceLocalizationService
assoc.getResource().handle(
new ResourceLocalizedEvent(key,
local, FileUtil.getDU(new File(local.toUri()))));
+ publicRsrc.localizationCompleted(key, true);
synchronized (attempts) {
attempts.remove(key);
}
} catch (ExecutionException e) {
LOG.info("Failed to download rsrc " + assoc.getResource(),
e.getCause());
+ LocalResourceRequest req = assoc.getResource().getRequest();
dispatcher.getEventHandler().handle(
new ContainerResourceFailedEvent(
assoc.getContext().getContainerId(),
- assoc.getResource().getRequest(), e.getCause()));
+ req, e.getCause()));
+ publicRsrc.localizationCompleted(req, false);
List<LocalizerResourceRequestEvent> reqs;
synchronized (attempts) {
- LocalResourceRequest req = assoc.getResource().getRequest();
reqs = attempts.get(req);
if (null == reqs) {
LOG.error("Missing pending list for " + req);
@@ -1003,4 +1033,4 @@ public class ResourceLocalizationService
del.delete(null, dirPath, new Path[] {});
}
-}
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java Fri Apr 5 02:43:29 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
/**
* This class allows a node manager to run without without communicating with a
@@ -73,9 +74,9 @@ public class MockNodeStatusUpdater exten
LOG.info("Got heartbeat number " + heartBeatID);
nodeStatus.setResponseId(heartBeatID++);
- NodeHeartbeatResponse nhResponse = recordFactory
- .newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setResponseId(heartBeatID);
+ NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils
+ .newNodeHeartbeatResponse(heartBeatID, null, null,
+ null, null, 1000L);
return nhResponse;
}
}