You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/24 19:07:23 UTC
[01/50] [abbrv] hadoop git commit: YARN-8791. Trim docker inspect
output for line feed for STOPSIGNAL parsing. Contributed by Chandni Singh
[Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-1011 4d858dd92 -> 954986dd5 (forced update)
YARN-8791. Trim docker inspect output for line feed for STOPSIGNAL parsing.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efdea85a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efdea85a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efdea85a
Branch: refs/heads/YARN-1011
Commit: efdea85ad1cd4cc5a2a306898dbdb2c14b952d02
Parents: 1824d5d
Author: Eric Yang <ey...@apache.org>
Authored: Wed Sep 19 13:16:11 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Sep 19 13:16:11 2018 -0400
----------------------------------------------------------------------
.../linux/runtime/DockerLinuxContainerRuntime.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efdea85a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 2b53f13..55dfdc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1282,7 +1282,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
DockerInspectCommand.STATUS_TEMPLATE,
DockerInspectCommand.STOPSIGNAL_TEMPLATE}, delimiter);
try {
- String output = executeDockerInspect(containerId, inspectCommand);
+ String output = executeDockerInspect(containerId, inspectCommand).trim();
if (!output.isEmpty()) {
String[] statusAndSignal = StringUtils.split(output, delimiter);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDDS-514. Clean Unregister JMX
upon SCMConnectionManager#close. Contributed by Xiaoyu Yao.
Posted by ha...@apache.org.
HDDS-514. Clean Unregister JMX upon SCMConnectionManager#close.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/524f7cd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/524f7cd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/524f7cd3
Branch: refs/heads/YARN-1011
Commit: 524f7cd354e0683c9ec61fdbce344ef79b841728
Parents: 096a716
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Sep 20 12:21:34 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Sep 20 12:21:34 2018 -0700
----------------------------------------------------------------------
.../container/common/statemachine/SCMConnectionManager.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/524f7cd3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
index 85fb580..775a91a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
@@ -59,7 +59,7 @@ public class SCMConnectionManager
private final int rpcTimeout;
private final Configuration conf;
- private final ObjectName jmxBean;
+ private ObjectName jmxBean;
public SCMConnectionManager(Configuration conf) {
this.mapLock = new ReentrantReadWriteLock();
@@ -191,7 +191,10 @@ public class SCMConnectionManager
public void close() throws IOException {
getValues().forEach(endpointStateMachine
-> IOUtils.cleanupWithLogger(LOG, endpointStateMachine));
- MBeans.unregister(jmxBean);
+ if (jmxBean != null) {
+ MBeans.unregister(jmxBean);
+ jmxBean = null;
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-4512 [YARN-1011]. Provide a
knob to turn on over-allocation. (kasha)
Posted by ha...@apache.org.
YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6389e0a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6389e0a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6389e0a2
Branch: refs/heads/YARN-1011
Commit: 6389e0a27bf662f7b732beda4383bc02aeebbbd4
Parents: 0cd6346
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Fri Jan 29 14:31:45 2016 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 14:10:13 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 13 ++-
.../src/main/resources/yarn-default.xml | 21 ++++
.../RegisterNodeManagerRequest.java | 14 ++-
.../pb/RegisterNodeManagerRequestPBImpl.java | 48 ++++++++-
.../server/api/records/OverAllocationInfo.java | 45 ++++++++
.../server/api/records/ResourceThresholds.java | 45 ++++++++
.../impl/pb/OverAllocationInfoPBImpl.java | 106 +++++++++++++++++++
.../impl/pb/ResourceThresholdsPBImpl.java | 93 ++++++++++++++++
.../yarn_server_common_service_protos.proto | 10 ++
.../hadoop/yarn/server/nodemanager/Context.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 17 +++
.../nodemanager/NodeStatusUpdaterImpl.java | 7 +-
.../monitor/ContainersMonitorImpl.java | 34 ++++++
.../amrmproxy/BaseAMRMProxyTest.java | 11 ++
14 files changed, 457 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a82801d..2e128a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2108,7 +2108,6 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
false;
-
// Configurations for applicaiton life time monitor feature
public static final String RM_APPLICATION_MONITOR_INTERVAL_MS =
RM_PREFIX + "application-timeouts.monitor.interval-ms";
@@ -2116,6 +2115,18 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
3000;
+ /** Overallocation (= allocation based on utilization) configs. */
+ public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD =
+ NM_PREFIX + "overallocation.allocation-threshold";
+ public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD
+ = 0f;
+ @Private
+ public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f;
+ public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold";
+ public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
+ = 0f;
+
/**
* Interval of time the linux container executor should try cleaning up
* cgroups entry when cleaning up a container. This is required due to what
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0700902..8001580 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1726,6 +1726,27 @@
</property>
<property>
+ <description>The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node,
+ expressed as a float between 0 and 0.95. By default, over-allocation is
+ turned off (value = 0). When turned on, the node allows running
+ OPPORTUNISTIC containers when the aggregate utilization is under the
+ value specified here multiplied by the node's advertised capacity.
+ </description>
+ <name>yarn.nodemanager.overallocation.allocation-threshold</name>
+ <value>0f</value>
+ </property>
+
+ <property>
+ <description>When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the utilization
+ beyond which OPPORTUNISTIC containers should start getting preempted.
+ </description>
+ <name>yarn.nodemanager.overallocation.preemption-threshold</name>
+ <value>1</value>
+ </property>
+
+ <property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
index ff50330..66c0a98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.util.Records;
public abstract class RegisterNodeManagerRequest {
@@ -42,14 +43,14 @@ public abstract class RegisterNodeManagerRequest {
List<NMContainerStatus> containerStatuses,
List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels) {
return newInstance(nodeId, httpPort, resource, nodeManagerVersionId,
- containerStatuses, runningApplications, nodeLabels, null);
+ containerStatuses, runningApplications, nodeLabels, null, null);
}
public static RegisterNodeManagerRequest newInstance(NodeId nodeId,
int httpPort, Resource resource, String nodeManagerVersionId,
List<NMContainerStatus> containerStatuses,
List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels,
- Resource physicalResource) {
+ Resource physicalResource, OverAllocationInfo overAllocationInfo) {
RegisterNodeManagerRequest request =
Records.newRecord(RegisterNodeManagerRequest.class);
request.setHttpPort(httpPort);
@@ -60,9 +61,10 @@ public abstract class RegisterNodeManagerRequest {
request.setRunningApplications(runningApplications);
request.setNodeLabels(nodeLabels);
request.setPhysicalResource(physicalResource);
+ request.setOverAllocationInfo(overAllocationInfo);
return request;
}
-
+
public abstract NodeId getNodeId();
public abstract int getHttpPort();
public abstract Resource getResource();
@@ -70,7 +72,11 @@ public abstract class RegisterNodeManagerRequest {
public abstract List<NMContainerStatus> getNMContainerStatuses();
public abstract Set<NodeLabel> getNodeLabels();
public abstract void setNodeLabels(Set<NodeLabel> nodeLabels);
-
+
+ public abstract OverAllocationInfo getOverAllocationInfo();
+ public abstract void setOverAllocationInfo(
+ OverAllocationInfo overAllocationInfo);
+
/**
* We introduce this here because currently YARN RM doesn't persist nodes info
* for application running. When RM restart happened, we cannot determinate if
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
index 02fd20f..802a9fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
@@ -42,12 +42,15 @@ import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregation
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto.Builder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
-
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.impl.pb.OverAllocationInfoPBImpl;
+
public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest {
RegisterNodeManagerRequestProto proto = RegisterNodeManagerRequestProto.getDefaultInstance();
RegisterNodeManagerRequestProto.Builder builder = null;
@@ -58,6 +61,7 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
private List<NMContainerStatus> containerStatuses = null;
private List<ApplicationId> runningApplications = null;
private Set<NodeLabel> labels = null;
+ private OverAllocationInfo overAllocationInfo = null;
private List<LogAggregationReport> logAggregationReportsForApps = null;
@@ -107,6 +111,11 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
if (this.logAggregationReportsForApps != null) {
addLogAggregationStatusForAppsToProto();
}
+
+ if (this.overAllocationInfo != null) {
+ builder.setOverAllocationInfo(
+ convertToProtoFormat(this.overAllocationInfo));
+ }
}
private void addLogAggregationStatusForAppsToProto() {
@@ -387,7 +396,30 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
builder.clearNodeLabels();
this.labels = nodeLabels;
}
-
+
+ @Override
+ public synchronized OverAllocationInfo getOverAllocationInfo() {
+ RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.overAllocationInfo != null) {
+ return this.overAllocationInfo;
+ }
+ if (!p.hasOverAllocationInfo()) {
+ return null;
+ }
+ this.overAllocationInfo = convertFromProtoFormat(p.getOverAllocationInfo());
+ return this.overAllocationInfo;
+ }
+
+ @Override
+ public synchronized void setOverAllocationInfo(
+ OverAllocationInfo overAllocationInfo) {
+ maybeInitBuilder();
+ if (this.overAllocationInfo == null) {
+ builder.clearOverAllocationInfo();
+ }
+ this.overAllocationInfo = overAllocationInfo;
+ }
+
private synchronized void initNodeLabels() {
if (this.labels != null) {
return;
@@ -475,9 +507,19 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
@Override
public synchronized void setLogAggregationReportsForApps(
List<LogAggregationReport> logAggregationStatusForApps) {
- if(logAggregationStatusForApps == null) {
+ if (logAggregationStatusForApps == null) {
builder.clearLogAggregationReportsForApps();
}
this.logAggregationReportsForApps = logAggregationStatusForApps;
}
+
+ private static OverAllocationInfoProto convertToProtoFormat(
+ OverAllocationInfo overAllocationInfo) {
+ return ((OverAllocationInfoPBImpl)overAllocationInfo).getProto();
+ }
+
+ private static OverAllocationInfo convertFromProtoFormat(
+ OverAllocationInfoProto proto) {
+ return new OverAllocationInfoPBImpl(proto);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
new file mode 100644
index 0000000..77952bf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.api.records.impl.pb
+ .OverAllocationInfoPBImpl;
+
+/**
+ * Captures information on how aggressively the scheduler can over-allocate
+ * OPPORTUNISTIC containers on a node. This is node-specific, and is sent on
+ * the wire on each heartbeat.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class OverAllocationInfo {
+ public static OverAllocationInfo newInstance(
+ ResourceThresholds overAllocationThresholds) {
+ OverAllocationInfo info = new OverAllocationInfoPBImpl();
+ info.setOverAllocationThreshold(overAllocationThresholds);
+ return info;
+ }
+
+ public abstract ResourceThresholds getOverAllocationThresholds();
+
+ public abstract void setOverAllocationThreshold(
+ ResourceThresholds resourceThresholds);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
new file mode 100644
index 0000000..d57706a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImpl;
+
+/**
+ * Captures resource thresholds to be used for allocation and preemption
+ * when over-allocation is turned on.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class ResourceThresholds {
+ public static ResourceThresholds newInstance(float threshold) {
+ ResourceThresholds thresholds = new ResourceThresholdsPBImpl();
+ thresholds.setMemoryThreshold(threshold);
+ thresholds.setCpuThreshold(threshold);
+ return thresholds;
+ }
+
+ public abstract float getMemoryThreshold();
+
+ public abstract float getCpuThreshold();
+
+ public abstract void setMemoryThreshold(float memoryThreshold);
+
+ public abstract void setCpuThreshold(float cpuThreshold);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
new file mode 100644
index 0000000..758f4fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProto;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+
+public class OverAllocationInfoPBImpl extends OverAllocationInfo {
+ private OverAllocationInfoProto proto =
+ OverAllocationInfoProto.getDefaultInstance();
+ private OverAllocationInfoProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ResourceThresholds overAllocationThresholds = null;
+
+ public OverAllocationInfoPBImpl() {
+ builder = OverAllocationInfoProto.newBuilder();
+ }
+
+ public OverAllocationInfoPBImpl(OverAllocationInfoProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized OverAllocationInfoProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ if (overAllocationThresholds != null) {
+ builder.setOverAllocationThresholds(
+ convertToProtoFormat(overAllocationThresholds));
+ }
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = OverAllocationInfoProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized ResourceThresholds getOverAllocationThresholds() {
+ OverAllocationInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (overAllocationThresholds != null) {
+ return overAllocationThresholds;
+ }
+ if (!p.hasOverAllocationThresholds()) {
+ return null;
+ }
+ overAllocationThresholds =
+ convertFromProtoFormat(p.getOverAllocationThresholds());
+ return overAllocationThresholds;
+ }
+
+ @Override
+ public synchronized void setOverAllocationThreshold(
+ ResourceThresholds resourceThresholds) {
+ maybeInitBuilder();
+ if (this.overAllocationThresholds != null) {
+ builder.clearOverAllocationThresholds();
+ }
+ this.overAllocationThresholds = resourceThresholds;
+ }
+
+ private static ResourceThresholdsProto convertToProtoFormat(
+ ResourceThresholds overAllocationThresholds) {
+ return ((ResourceThresholdsPBImpl) overAllocationThresholds).getProto();
+ }
+
+ private static ResourceThresholds convertFromProtoFormat(
+ ResourceThresholdsProto overAllocationThresholdsProto) {
+ return new ResourceThresholdsPBImpl(overAllocationThresholdsProto);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
new file mode 100644
index 0000000..10fb284
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+
+public class ResourceThresholdsPBImpl extends ResourceThresholds{
+ private ResourceThresholdsProto proto =
+ ResourceThresholdsProto.getDefaultInstance();
+ private ResourceThresholdsProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public ResourceThresholdsPBImpl() {
+ builder = ResourceThresholdsProto.newBuilder();
+ }
+
+ public ResourceThresholdsPBImpl(ResourceThresholdsProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized ResourceThresholdsProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ /*
+ * Right now, we have only memory and cpu thresholds that are floats.
+ * This is a no-op until we add other non-static fields to the proto.
+ */
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceThresholdsProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized float getMemoryThreshold() {
+ ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemory();
+ }
+
+ @Override
+ public synchronized float getCpuThreshold() {
+ ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getCpu();
+ }
+
+ @Override
+ public synchronized void setMemoryThreshold(float memoryThreshold) {
+ maybeInitBuilder();
+ builder.setMemory(memoryThreshold);
+ }
+
+ @Override
+ public synchronized void setCpuThreshold(float cpuThreshold) {
+ maybeInitBuilder();
+ builder.setCpu(cpuThreshold);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 0b8c4a3..50d72f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -72,6 +72,7 @@ message RegisterNodeManagerRequestProto {
optional NodeLabelsProto nodeLabels = 8;
optional ResourceProto physicalResource = 9;
repeated LogAggregationReportProto log_aggregation_reports_for_apps = 10;
+ optional OverAllocationInfoProto overAllocationInfo = 11;
}
message RegisterNodeManagerResponseProto {
@@ -203,3 +204,12 @@ message SCMUploaderCanUploadRequestProto {
message SCMUploaderCanUploadResponseProto {
optional bool uploadable = 1;
}
+
+message OverAllocationInfoProto {
+ optional ResourceThresholdsProto over_allocation_thresholds = 1;
+}
+
+message ResourceThresholdsProto {
+ optional float memory = 1 [default = 0];
+ optional float cpu = 2 [default = 0];
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 84b3915..75d06c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
@@ -137,4 +138,8 @@ public interface Context {
* @return the NM {@code DeletionService}.
*/
DeletionService getDeletionService();
+
+ boolean isOverAllocationEnabled();
+
+ OverAllocationInfo getOverAllocationInfo();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 6eda4a8..af767a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
@@ -684,6 +685,8 @@ public class NodeManager extends CompositeService
private NMLogAggregationStatusTracker nmLogAggregationStatusTracker;
+ private OverAllocationInfo overAllocationInfo;
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
@@ -831,6 +834,20 @@ public class NodeManager extends CompositeService
this.nodeStatusUpdater = nodeStatusUpdater;
}
+ @Override
+ public boolean isOverAllocationEnabled() {
+ return getOverAllocationInfo() != null;
+ }
+
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return this.overAllocationInfo;
+ }
+
+ public void setOverAllocationInfo(OverAllocationInfo overAllocationInfo) {
+ this.overAllocationInfo = overAllocationInfo;
+ }
+
public boolean isDistributedSchedulingEnabled() {
return isDistSchedulingEnabled;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 3bb9f92..572684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -387,8 +387,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
RegisterNodeManagerRequest request =
RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource,
nodeManagerVersionId, containerReports, getRunningApplications(),
- nodeLabels, physicalResource);
-
+ nodeLabels, physicalResource, context.getOverAllocationInfo());
if (containerReports != null) {
LOG.info("Registering with RM using containers :" + containerReports);
}
@@ -519,8 +518,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
= getIncreasedContainers();
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
- createKeepAliveApplicationList(), nodeHealthStatus,
- containersUtilization, nodeUtilization, increasedContainers);
+ createKeepAliveApplicationList(), nodeHealthStatus,
+ containersUtilization, nodeUtilization, increasedContainers);
nodeStatus.setOpportunisticContainersStatus(
getOpportunisticContainersStatus());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index d83fe39..cdd3476 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -38,8 +38,11 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
@@ -110,6 +113,8 @@ public class ContainersMonitorImpl extends AbstractService implements
private ResourceUtilization containersUtilization;
+ private ResourceThresholds overAllocationPreemptionThresholds;
+
private volatile boolean stopped = false;
public ContainersMonitorImpl(ContainerExecutor exec,
@@ -213,6 +218,13 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
+ initializeOverAllocation(conf);
+ if (context.isOverAllocationEnabled()) {
+ pmemCheckEnabled = true;
+ LOG.info("Force enabling physical memory checks because " +
+ "overallocation is enabled");
+ }
+
containersMonitorEnabled =
isContainerMonitorEnabled() && monitoringInterval > 0;
LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled);
@@ -264,6 +276,28 @@ public class ContainersMonitorImpl extends AbstractService implements
pId, processTreeClass, conf);
}
+ private void initializeOverAllocation(Configuration conf) {
+ float overAllocationTreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
+ overAllocationTreshold = Math.min(overAllocationTreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
+ overAllocationTreshold = Math.max(0, overAllocationTreshold);
+
+ if (overAllocationTreshold > 0f) {
+ ((NodeManager.NMContext) context).setOverAllocationInfo(
+ OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(overAllocationTreshold)));
+
+ float preemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
+
+ this.overAllocationPreemptionThresholds =
+ ResourceThresholds.newInstance(preemptionThreshold);
+ }
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6389e0a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 2794857..94f8bb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@@ -772,6 +773,16 @@ public abstract class BaseAMRMProxyTest {
}
@Override
+ public boolean isOverAllocationEnabled() {
+ return false;
+ }
+
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
+ @Override
public NodeResourceMonitor getNodeResourceMonitor() {
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: Revert "YARN-6675. Add NM support
to launch opportunistic containers based on overallocation. Contributed by
Haibo Chen."
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index f24df38..6b3ac67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -67,8 +66,8 @@ public class TestContainerSchedulerRecovery {
@Mock private ContainerId containerId;
- @Mock private AllocationBasedResourceTracker
- allocationBasedResourceTracker;
+ @Mock private AllocationBasedResourceUtilizationTracker
+ allocationBasedResourceUtilizationTracker;
@InjectMocks private ContainerScheduler tempContainerScheduler =
new ContainerScheduler(context, dispatcher, metrics, 0);
@@ -86,13 +85,12 @@ public class TestContainerSchedulerRecovery {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
when(container.getContainerId()).thenReturn(containerId);
- when(container.getResource()).thenReturn(Resource.newInstance(1024, 1));
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(containerId.getApplicationAttemptId().getApplicationId())
.thenReturn(appId);
when(containerId.getContainerId()).thenReturn(123L);
- doNothing().when(allocationBasedResourceTracker)
- .containerLaunched(container);
+ doNothing().when(allocationBasedResourceUtilizationTracker)
+ .addContainerResources(container);
}
@After public void tearDown() {
@@ -114,8 +112,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
@@ -134,8 +132,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as PAUSED, GUARANTEED,
@@ -154,8 +152,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
@@ -174,8 +172,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as LAUNCHED, GUARANTEED,
@@ -194,8 +192,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
+ .addContainerResources(container);
}
/*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
@@ -214,8 +212,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
+ .addContainerResources(container);
}
/*Test if a container is recovered as REQUESTED, GUARANTEED,
@@ -234,8 +232,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
@@ -254,8 +252,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as COMPLETED, GUARANTEED,
@@ -274,8 +272,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
@@ -294,8 +292,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as GUARANTEED but no executionType set,
@@ -313,8 +311,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as PAUSED but no executionType set,
@@ -332,7 +330,7 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
deleted file mode 100644
index 384b116..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
+++ /dev/null
@@ -1,1121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ContainerSubState;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.exceptions.ConfigurationException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
-import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-
-/**
- * Test ContainerScheduler behaviors when NM overallocation is turned on.
- */
-public class TestContainerSchedulerWithOverAllocation
- extends BaseContainerManagerTest {
- private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
- private static final int NM_CONTAINERS_VCORES = 4;
- private static final int NM_CONTAINERS_MEMORY_MB = 2048;
-
- static {
- LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
- }
-
- public TestContainerSchedulerWithOverAllocation()
- throws UnsupportedFileSystemException {
- }
-
- @Override
- protected ContainerExecutor createContainerExecutor() {
- DefaultContainerExecutor exec =
- new LongRunningContainerSimulatingContainerExecutor();
- exec.setConf(conf);
- return exec;
- }
-
- @Override
- protected ContainerManagerImpl createContainerManager(
- DeletionService delSrvc) {
- return new LongRunningContainerSimulatingContainersManager(
- context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler, user);
- }
-
- @Override
- public void setup() throws IOException {
- conf.setInt(
- YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
- NM_OPPORTUNISTIC_QUEUE_LIMIT);
- conf.setFloat(
- YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
- 0.75f);
- conf.setFloat(
- YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
- 0.75f);
- super.setup();
- }
-
- /**
- * Start one GUARANTEED and one OPPORTUNISTIC container, which in aggregate do
- * not exceed the capacity of the node. Both containers are expected to start
- * running immediately.
- */
- @Test
- public void testStartMultipleContainersWithoutOverallocation()
- throws Exception {
- containerManager.start();
-
- StartContainersRequest allRequests = StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() { {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
- } }
- );
- containerManager.startContainers(allRequests);
-
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(1), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start one GUARANTEED and one OPPORTUNISTIC containers whose utilization
- * is very low relative to their resource request, resulting in a low node
- * utilization. Then start another OPPORTUNISTIC containers which requests
- * more than what's left unallocated on the node. Due to overallocation
- * being turned on and node utilization being low, the second OPPORTUNISTIC
- * container is also expected to be launched immediately.
- */
- @Test
- public void testStartOppContainersWithPartialOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(1), ContainerSubState.RUNNING);
-
- // the current containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(512, 0, 1.0f/8));
-
- // start a container that requests more than what's left unallocated
- // 512 + 1024 + 824 > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
- ));
-
- // this container is expected to be started immediately because there
- // are (memory: 1024, vcore: 0.625) available based on over-allocation
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start one GUARANTEED and one OPPORTUNISTIC containers which utilizes most
- * of the resources they requested, resulting in a high node utilization.
- * Then start another OPPORTUNISTIC containers which requests more than what's
- * left unallocated on the node. Because of the high resource utilization on
- * the node, the projected utilization, if we were to start the second
- * OPPORTUNISTIC container immediately, will go over the NM overallocation
- * threshold, so the second OPPORTUNISTIC container is expected to be queued.
- */
- @Test
- public void testQueueOppContainerWithPartialOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(1), ContainerSubState.RUNNING);
-
- // the containers utilization is high
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1500, 0, 1.0f/8));
-
- // start a container that requests more than what's left unallocated
- // 512 + 1024 + 824 > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
- ));
- // this container will not start immediately because there is not
- // enough resource available at the moment either in terms of
- // resources unallocated or in terms of the actual utilization
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.SCHEDULED);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.SCHEDULED);
- }
- });
- }
-
- /**
- * Start two GUARANTEED containers which in aggregate takes up the whole node
- * capacity, yet whose utilization is low relative to their resource request,
- * resulting in a low node resource utilization. Then try to start another
- * OPPORTUNISTIC containers. Because the resource utilization across the node
- * is low and overallocation being turned on, the OPPORTUNISTIC container is
- * expected to be launched immediately even though there is no resources left
- * unallocated.
- */
- @Test
- public void testStartOppContainersWithOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // the current containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(800, 0, 1.0f/8));
-
- // start a container when there is no resources left unallocated.
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
- ));
-
- // this container is expected to be started because there is resources
- // available because the actual utilization is very low
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
-
- /**
- * Start two GUARANTEED containers which in aggregate take up the whole node
- * capacity and fully utilize the resources they requested. Then try to start
- * four OPPORTUNISTIC containers of which three will be queued and one will be
- * killed because of the max queue length is 3.
- */
- @Test
- public void testQueueOppContainersWithFullUtilization() throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // the containers are fully utilizing their resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(2048, 0, 1.0f/8));
-
- // start more OPPORTUNISTIC containers than what the OPPORTUNISTIC container
- // queue can hold when there is no unallocated resource left.
- List<StartContainerRequest> moreContainerRequests =
- new ArrayList<>(NM_OPPORTUNISTIC_QUEUE_LIMIT + 1);
- for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
- moreContainerRequests.add(
- createStartContainerRequest(2 + a,
- BuilderUtils.newResource(512, 1), false));
- }
- containerManager.startContainers(
- StartContainersRequest.newInstance(moreContainerRequests));
-
- // All OPPORTUNISTIC containers but the last one should be queued.
- // The last OPPORTUNISTIC container to launch should be killed.
- BaseContainerManagerTest.waitForContainerState(
- containerManager, createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT + 2),
- ContainerState.COMPLETE);
-
- HashMap<ContainerId, ContainerSubState> expectedContainerStatus =
- new HashMap<>();
- expectedContainerStatus.put(
- createContainerId(0), ContainerSubState.RUNNING);
- expectedContainerStatus.put(
- createContainerId(1), ContainerSubState.RUNNING);
- expectedContainerStatus.put(
- createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT),
- ContainerSubState.DONE);
- for (int i = 0; i < NM_OPPORTUNISTIC_QUEUE_LIMIT; i++) {
- expectedContainerStatus.put(
- createContainerId(i + 2), ContainerSubState.SCHEDULED);
- }
- verifyContainerStatuses(expectedContainerStatus);
- }
-
- /**
- * Start two GUARANTEED containers that together does not take up the
- * whole node. Then try to start one OPPORTUNISTIC container that will
- * fit into the remaining unallocated space on the node.
- * The OPPORTUNISTIC container is expected to start even though the
- * current node utilization is above the NM overallocation threshold,
- * because it's always safe to launch containers as long as the node
- * has not been fully allocated.
- */
- @Test
- public void testStartOppContainerWithHighUtilizationNoOverallocation()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1200, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(400, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // containers utilization is above the over-allocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1600, 0, 1.0f/2));
-
- // start a container that can just fit in the remaining unallocated space
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(400, 1), false))
- ));
-
- // the OPPORTUNISTIC container can be safely launched even though
- // the container utilization is above the NM overallocation threshold
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start two OPPORTUNISTIC containers first whose utilization is low relative
- * to the resources they requested, resulting in a low node utilization. Then
- * try to start a GUARANTEED container which requests more than what's left
- * unallocated on the node. Because the node utilization is low and NM
- * overallocation is turned on, the GUARANTEED container is expected to be
- * started immediately without killing any running OPPORTUNISTIC containers.
- */
- @Test
- public void testKillNoOppContainersWithPartialOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(512, 0, 1.0f/8));
-
- // start a GUARANTEED container that requests more than what's left
- // unallocated on the node: (512 + 1024 + 824) > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
- ));
-
- // the GUARANTEED container is expected be launched immediately without
- // killing any OPPORTUNISTIC containers.
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start two OPPORTUNISTIC containers whose utilization will be high relative
- * to the resources they requested, resulting in a high node utilization.
- * Then try to start a GUARANTEED container which requests more than what's
- * left unallocated on the node. Because the node is under high utilization,
- * the second OPPORTUNISTIC container is expected to be killed in order to
- * make room for the GUARANTEED container.
- */
- @Test
- public void testKillOppContainersWithPartialOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // the containers utilization is very high
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1800, 0, 1.0f/8));
-
- // start a GUARANTEED container that requests more than what's left
- // unallocated on the node 512 + 1024 + 824 > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
- ));
-
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
- // the last launched OPPORTUNISTIC container is expected to be killed
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.DONE);
-
- GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
- newInstance(new ArrayList<ContainerId>() {
- {
- add(createContainerId(0));
- add(createContainerId(1));
- add(createContainerId(2));
- }
- });
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
- for (ContainerStatus status : containerStatuses) {
- if (status.getContainerId().equals(createContainerId(1))) {
- Assert.assertTrue(status.getDiagnostics().contains(
- "Container Killed to make room for Guaranteed Container"));
- } else {
- Assert.assertEquals(status.getContainerId() + " is not RUNNING",
- ContainerSubState.RUNNING, status.getContainerSubState());
- }
- System.out.println("\nStatus : [" + status + "]\n");
- }
- }
-
-
- /**
- * Start three OPPORTUNISTIC containers which in aggregates exceeds the
- * capacity of the node, yet whose utilization is low relative
- * to the resources they requested, resulting in a low node utilization.
- * Then try to start a GUARANTEED container. Even though the node has
- * nothing left unallocated, it is expected to start immediately
- * without killing any running OPPORTUNISTIC containers because the node
- * utilization is very low and overallocation is turned on.
- */
- @Test
- public void testKillNoOppContainersWithOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), false));
- }
- }
- ));
- // All three GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- // the containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1024, 0, 1.0f/8));
-
- // start a GUARANTEED container that requests more than what's left
- // unallocated on the node: (512 + 1024 + 824) > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), true))
- ));
-
- // the GUARANTEED container is expected be launched immediately without
- // killing any OPPORTUNISTIC containers
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- put(createContainerId(3), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start four OPPORTUNISTIC containers which in aggregates exceeds the
- * capacity of the node. The real resource utilization of the first two
- * OPPORTUNISTIC containers are high whereas that of the latter two are
- * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
- * container will eventually start running after preempting the third
- * and fourth OPPORTUNISTIC container (which releases no resources) and
- * then the second OPPORTUNISTIC container.
- */
- @Test
- public void
- testKillOppContainersConservativelyWithOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(3,
- BuilderUtils.newResource(1024, 1), false));
- }
- }
- ));
- // All four GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
-
- // the containers utilization is at the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- // try to start a GUARANTEED container when there's nothing left unallocated
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(4,
- BuilderUtils.newResource(1024, 1), true))
- ));
-
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
- GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
- newInstance(new ArrayList<ContainerId>() {
- {
- add(createContainerId(0));
- add(createContainerId(1));
- add(createContainerId(2));
- add(createContainerId(3));
- add(createContainerId(4));
- }
- });
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
- for (ContainerStatus status : containerStatuses) {
- if (status.getContainerId().equals(createContainerId(0)) ||
- status.getContainerId().equals(createContainerId(4))) {
- Assert.assertEquals(
- ContainerSubState.RUNNING, status.getContainerSubState());
- } else {
- Assert.assertTrue(status.getDiagnostics().contains(
- "Container Killed to make room for Guaranteed Container"));
- }
- System.out.println("\nStatus : [" + status + "]\n");
- }
- }
-
- /**
- * Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
- * which in aggregate exceeds the capacity of the node. The first two
- * OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
- * one utilizes nearly all of its resource requested. Then try to start two
- * more OPPORTUNISTIC containers. The two OPPORTUNISTIC containers are
- * expected to be queued immediately. Upon the completion of the
- * resource-usage-heavy GUARANTEED container, both OPPORTUNISTIC containers
- * are expected to start.
- */
- @Test
- public void testStartOppContainersUponContainerCompletion() throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), true));
- }
- }
- ));
-
- // All three containers are all expected to start immediately
- // because the node utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- // the contianers utilization is at the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(4,
- BuilderUtils.newResource(512, 1), false));
- }
- }
- ));
- // the two new OPPORTUNISTIC containers are expected to be queued
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(3), ContainerSubState.SCHEDULED);
- put(createContainerId(4), ContainerSubState.SCHEDULED);
- }
- });
-
- // the GUARANTEED container is completed releasing resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(100, 0, 1.0f/5));
- allowContainerToSucceed(2);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.DONE);
-
- // the two OPPORTUNISTIC containers are expected to start together
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.DONE);
- put(createContainerId(3), ContainerSubState.RUNNING);
- put(createContainerId(4), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start one GUARANTEED container that consumes all the resources on the
- * node and keeps running, followed by two OPPORTUNISTIC containers that
- * will be queued forever because there is no containers starting or
- * finishing. Then try to start OPPORTUNISTIC containers out of band.
- */
- @Test
- public void testStartOpportunisticContainersOutOfBand() throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(0,
- BuilderUtils.newResource(2048, 4), true))));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
-
- // the container is fully utilizing its resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(2048, 0, 1.0f));
-
- // send two OPPORTUNISTIC container requests that are expected to be queued
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.SCHEDULED);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.SCHEDULED);
-
- // the containers utilization dropped to the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- // try to start containers out of band.
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
-
- // no containers in queue are expected to be launched because the
- // containers utilization is not below the over-allocation threshold
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.SCHEDULED);
- put(createContainerId(2), ContainerSubState.SCHEDULED);
- }
- });
-
- // the GUARANTEED container is completed releasing resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(100, 0, 1.0f/5));
-
- // the containers utilization dropped way below the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(512, 0, 1.0f/8));
-
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
-
- // the two OPPORTUNISTIC containers are expected to be launched
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
-
- private void setContainerResourceUtilization(ResourceUtilization usage) {
- ((ContainerMonitorForOverallocationTest)
- containerManager.getContainersMonitor())
- .setContainerResourceUsage(usage);
- }
-
- private void allowContainerToSucceed(int containerId) {
- ((LongRunningContainerSimulatingContainerExecutor) this.exec)
- .containerSucceeded(createContainerId(containerId));
- }
-
-
- protected StartContainerRequest createStartContainerRequest(int containerId,
- Resource resource, boolean isGuaranteed) throws IOException {
- ContainerLaunchContext containerLaunchContext =
- recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
- ExecutionType.OPPORTUNISTIC;
- Token containerToken = createContainerToken(
- createContainerId(containerId),
- DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
- context.getContainerTokenSecretManager(),
- null, executionType);
-
- return StartContainerRequest.newInstance(
- containerLaunchContext, containerToken);
- }
-
- protected void verifyContainerStatuses(
- Map<ContainerId, ContainerSubState> expected)
- throws IOException, YarnException {
- List<ContainerId> statList = new ArrayList<>(expected.keySet());
- GetContainerStatusesRequest statRequest =
- GetContainerStatusesRequest.newInstance(statList);
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
-
- for (ContainerStatus status : containerStatuses) {
- ContainerId containerId = status.getContainerId();
- Assert.assertEquals(containerId + " is in unexpected state",
- expected.get(containerId), status.getContainerSubState());
- }
- }
-
- /**
- * A container manager that sends a dummy container pid while it's cleaning
- * up running containers. Used along with
- * LongRunningContainerSimulatingContainerExecutor to simulate long running
- * container processes for testing purposes.
- */
- private static class LongRunningContainerSimulatingContainersManager
- extends ContainerManagerImpl {
-
- private final String user;
-
- LongRunningContainerSimulatingContainersManager(
- Context context, ContainerExecutor exec,
- DeletionService deletionContext,
- NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics,
- LocalDirsHandlerService dirsHandler, String user) {
- super(context, exec, deletionContext,
- nodeStatusUpdater, metrics, dirsHandler);
- this.user = user;
- }
-
- @Override
- protected UserGroupInformation getRemoteUgi() throws YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
- .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
- .getKeyId()));
- return ugi;
- }
-
- /**
- * Create a container launcher that signals container processes
- * with a dummy pid. The container processes are simulated in
- * LongRunningContainerSimulatingContainerExecutor which does
- * not write a pid file on behalf of containers to launch, so
- * the pid does not matter.
- */
- @Override
- protected ContainersLauncher createContainersLauncher(
- Context context, ContainerExecutor exec) {
- ContainerManagerImpl containerManager = this;
- return new ContainersLauncher(context, dispatcher, exec, dirsHandler,
- this) {
- @Override
- protected ContainerLaunch createContainerLaunch(
- Application app, Container container) {
- return new ContainerLaunch(context, getConfig(), dispatcher,
- exec, app, container, dirsHandler, containerManager) {
- @Override
- protected String getContainerPid(Path pidFilePath)
- throws Exception {
- return "123";
- }
-
- };
- }
- };
- }
-
- @Override
- protected ContainersMonitor createContainersMonitor(
- ContainerExecutor exec) {
- return new ContainerMonitorForOverallocationTest(exec,
- dispatcher, context);
- }
-
- public void startContainersOutOfBandUponLowUtilization() {
- ((ContainerMonitorForOverallocationTest) getContainersMonitor())
- .attemptToStartContainersUponLowUtilization();
- }
- }
-
- /**
- * A container executor that simulates long running container processes
- * by having container launch threads sleep infinitely until it's given
- * a signal to finish with either a success or failure exit code.
- */
- private static class LongRunningContainerSimulatingContainerExecutor
- extends DefaultContainerExecutor {
- private ConcurrentHashMap<ContainerId, ContainerFinishLatch> containers =
- new ConcurrentHashMap<>();
-
- public void containerSucceeded(ContainerId containerId) {
- ContainerFinishLatch containerFinishLatch = containers.get(containerId);
- if (containerFinishLatch != null) {
- containerFinishLatch.toSucceed();
- }
- }
-
- public void containerFailed(ContainerId containerId) {
- ContainerFinishLatch containerFinishLatch = containers.get(containerId);
- if (containerFinishLatch != null) {
- containerFinishLatch.toFail();
- }
- }
-
- /**
- * Simulate long running container processes by having container launcher
- * threads wait infinitely for a signal to finish.
- */
- @Override
- public int launchContainer(ContainerStartContext ctx)
- throws IOException, ConfigurationException {
- ContainerId container = ctx.getContainer().getContainerId();
- containers.putIfAbsent(container, new ContainerFinishLatch(container));
-
- // simulate a long running container process by having the
- // container launch thread sleep forever until it's given a
- // signal to finish with a exit code.
- while (!containers.get(container).toProceed) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- return -1;
- }
- }
-
- return containers.get(container).getContainerExitCode();
- }
-
- /**
- * Override signalContainer() so that simulated container processes
- * are properly cleaned up.
- */
- @Override
- public boolean signalContainer(ContainerSignalContext ctx)
- throws IOException {
- containerSucceeded(ctx.getContainer().getContainerId());
- return true;
- }
-
- /**
- * A signal that container launch threads wait for before exiting
- * in order to simulate long running container processes.
- */
- private static final class ContainerFinishLatch {
- volatile boolean toProceed;
- int exitCode;
- ContainerId container;
-
- ContainerFinishLatch(ContainerId containerId) {
- exitCode = 0;
- toProceed = false;
- container = containerId;
- }
-
- void toSucceed() {
- exitCode = 0;
- toProceed = true;
- }
-
- void toFail() {
- exitCode = -101;
- toProceed = true;
- }
-
- int getContainerExitCode() {
- // read barrier of toProceed to make sure the exit code is not stale
- if (toProceed) {
- LOG.debug(container + " finished with exit code: " + exitCode);
- }
- return exitCode;
- }
- }
- }
-
- /**
- * A test implementation of container monitor that allows control of
- * current resource utilization.
- */
- private static class ContainerMonitorForOverallocationTest
- extends ContainersMonitorImpl {
-
- private ResourceUtilization containerResourceUsage =
- ResourceUtilization.newInstance(0, 0, 0.0f);
-
- ContainerMonitorForOverallocationTest(ContainerExecutor exec,
- AsyncDispatcher dispatcher, Context context) {
- super(exec, dispatcher, context);
- }
-
- @Override
- public long getPmemAllocatedForContainers() {
- return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
- }
-
- @Override
- public long getVmemAllocatedForContainers() {
- float pmemRatio = getConfig().getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- return (long) (pmemRatio * getPmemAllocatedForContainers());
- }
-
- @Override
- public long getVCoresAllocatedForContainers() {
- return NM_CONTAINERS_VCORES;
- }
-
- @Override
- public ContainersResourceUtilization getContainersUtilization(
- boolean latest) {
- return new ContainersMonitor.ContainersResourceUtilization(
- containerResourceUsage, System.currentTimeMillis());
- }
-
- public void setContainerResourceUsage(
- ResourceUtilization containerResourceUsage) {
- this.containerResourceUsage = containerResourceUsage;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-15756. [JDK10] Migrate from
sun.net.util.IPAddressUtil to the replacement. Contributed by Akira Ajisaka.
Posted by ha...@apache.org.
HADOOP-15756. [JDK10] Migrate from sun.net.util.IPAddressUtil to the replacement. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3da94a36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3da94a36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3da94a36
Branch: refs/heads/YARN-1011
Commit: 3da94a36e21a315c09ec7edb7702820fe2b524f9
Parents: 646874c
Author: Ewan Higgs <ew...@wdc.com>
Authored: Thu Sep 20 14:53:21 2018 +0200
Committer: Ewan Higgs <ew...@wdc.com>
Committed: Thu Sep 20 14:53:21 2018 +0200
----------------------------------------------------------------------
.../org/apache/hadoop/security/SecurityUtil.java | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3da94a36/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 5f8cb29..0de334a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -54,9 +54,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
//this will need to be replaced someday when there is a suitable replacement
import sun.net.dns.ResolverConfiguration;
-import sun.net.util.IPAddressUtil;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.InetAddresses;
/**
* Security Utils.
@@ -604,14 +604,11 @@ public final class SecurityUtil {
public InetAddress getByName(String host) throws UnknownHostException {
InetAddress addr = null;
- if (IPAddressUtil.isIPv4LiteralAddress(host)) {
- // use ipv4 address as-is
- byte[] ip = IPAddressUtil.textToNumericFormatV4(host);
- addr = InetAddress.getByAddress(host, ip);
- } else if (IPAddressUtil.isIPv6LiteralAddress(host)) {
- // use ipv6 address as-is
- byte[] ip = IPAddressUtil.textToNumericFormatV6(host);
- addr = InetAddress.getByAddress(host, ip);
+ if (InetAddresses.isInetAddress(host)) {
+ // valid ip address. use it as-is
+ addr = InetAddresses.forString(host);
+ // set hostname
+ addr = InetAddress.getByAddress(host, addr.getAddress());
} else if (host.endsWith(".")) {
// a rooted host ends with a dot, ex. "host."
// rooted hosts never use the search path, so only try an exact lookup
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-8628. [UI2] Few duplicated or
inconsistent information displayed in UI2. Contributed by Akhil PB.
Posted by ha...@apache.org.
YARN-8628. [UI2] Few duplicated or inconsistent information displayed in UI2. Contributed by Akhil PB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2752779
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2752779
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2752779
Branch: refs/heads/YARN-1011
Commit: a2752779ac1545f5e0a52fce3cff02a7007e95fb
Parents: 524f7cd
Author: Sunil G <su...@apache.org>
Authored: Fri Sep 21 15:47:10 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Fri Sep 21 15:47:10 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/controllers/yarn-app/components.js | 2 +-
.../webapp/app/controllers/yarn-component-instance/info.js | 5 +++--
.../webapp/app/controllers/yarn-component-instances/info.js | 3 ++-
.../main/webapp/app/routes/yarn-component-instance/info.js | 4 ++--
.../main/webapp/app/serializers/yarn-component-instance.js | 1 -
.../src/main/webapp/app/serializers/yarn-container.js | 2 +-
.../src/main/webapp/app/serializers/yarn-service-component.js | 2 +-
.../main/webapp/app/serializers/yarn-timeline-container.js | 2 +-
.../src/main/webapp/app/templates/yarn-app/configs.hbs | 7 ++++---
.../webapp/app/templates/yarn-component-instance/info.hbs | 4 ----
10 files changed, 15 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index 5981eb5..5a6c616 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -41,7 +41,7 @@ export default Ember.Controller.extend({
getCellContent: function(row) {
return {
displayText: row.get('name'),
- href: `#/yarn-component-instances/${row.get('name')}/info?service=${service}&&appid=${appId}`
+ href: `#/yarn-component-instances/${row.get('name')}/info?service=${service}&appid=${appId}`
};
}
}, {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance/info.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance/info.js
index e3abcb7..e920aa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance/info.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance/info.js
@@ -19,7 +19,8 @@
import Ember from 'ember';
export default Ember.Controller.extend({
- queryParams: ["appid", "service"],
+ queryParams: ["appid", "service", "containerid"],
appid: undefined,
- service: undefined
+ service: undefined,
+ containerid: undefined
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
index 44cfe17..be4b4f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
@@ -42,9 +42,10 @@ export default Ember.Controller.extend({
getCellContent: function(row) {
var component = row.get('component');
var instance = row.get('instanceName');
+ var containerId = row.get('containerId');
return {
text: instance,
- href: `#/yarn-component-instance/${component}/instances/${instance}/info?appid=${appId}&&service=${serviceName}`
+ href: `#/yarn-component-instance/${component}/instances/${instance}/info?appid=${appId}&service=${serviceName}&containerid=${containerId}`
};
}
}, {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
index 3753c75..a67324a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
@@ -29,8 +29,8 @@ export default AbstractRoute.extend({
componentName: params.component_name,
instanceName: instanceName,
container: this.store.query('yarn-component-instance', {appId: params.appid}).then(function(instances) {
- if (instances && instances.findBy('instanceName', instanceName)) {
- return instances.findBy('instanceName', instanceName);
+ if (instances && instances.findBy('containerId', params.containerid)) {
+ return instances.findBy('containerId', params.containerid);
}
return null;
}, function() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
index 1bd3b2c..ce0dfb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
@@ -34,7 +34,6 @@ export default DS.JSONAPISerializer.extend({
startedTimestamp: info.LAUNCH_TIME,
host: info.HOSTNAME,
node: info.BARE_HOST,
- hostUrl: 'N/A',
ipAddr: info.IP,
exitStatusCode: info.EXIT_STATUS_CODE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-container.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-container.js
index fc640c5..f7f8272 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-container.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-container.js
@@ -34,7 +34,7 @@ export default DS.JSONAPISerializer.extend({
finishedTime: Converter.timeStampToDate(payload.finishedTime),
elapsedTime: payload.elapsedTime,
logUrl: payload.logUrl,
- containerExitStatus: payload.containerExitStatus,
+ containerExitStatus: payload.containerExitStatus + '',
containerState: payload.containerState,
nodeId : payload.nodeId,
nodeHttpAddress: payload.nodeHttpAddress
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-service-component.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-service-component.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-service-component.js
index b0261fc..9dd7e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-service-component.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-service-component.js
@@ -52,7 +52,7 @@ export default DS.JSONAPISerializer.extend({
vcores: info.RESOURCE_CPU,
memory: info.RESOURCE_MEMORY,
priority: 'N/A',
- instances: 'N/A',
+ instances: '0',
createdTimestamp: payload.createdtime,
configs: newConfigs,
metrics: newMetrics
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
index 99ab6c4..5b62b2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-timeline-container.js
@@ -34,7 +34,7 @@ export default DS.JSONAPISerializer.extend({
startedTime: Converter.timeStampToDate(payload.createdtime),
finishedTime: Converter.timeStampToDate(payload.info.YARN_CONTAINER_FINISHED_TIME),
nodeHttpAddress: payload.info.YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS,
- containerExitStatus: payload.info.YARN_CONTAINER_EXIT_STATUS,
+ containerExitStatus: payload.info.YARN_CONTAINER_EXIT_STATUS + '',
containerState: payload.info.YARN_CONTAINER_STATE,
nodeId: payload.info.YARN_CONTAINER_ALLOCATED_HOST + ':' + payload.info.YARN_CONTAINER_ALLOCATED_PORT,
diagnosticsInfo: payload.info.YARN_CONTAINER_DIAGNOSTICS_INFO
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/configs.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/configs.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/configs.hbs
index 5f4d29d..daf9549 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/configs.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/configs.hbs
@@ -16,10 +16,11 @@
* limitations under the License.
}}
-<div class="col-md-12">
- {{metrics-table metrics=model.metrics type="Service"}}
+<div class="col-md-12" style="margin-bottom: 15px;">
+ <div class="col-md-12">
+ {{metrics-table metrics=model.metrics type="Service"}}
+ </div>
</div>
-
{{#if model.configs}}
<div class="col-md-12">
<div class="panel panel-default">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2752779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 553f4e8..1b9d04a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -50,10 +50,6 @@
<td>{{check-availability model.container.host}}</td>
</tr>
<tr>
- <td>Host URL</td>
- <td>{{check-availability model.container.hostUrl}}</td>
- </tr>
- <tr>
<td>Node</td>
<td>{{check-availability model.container.node}}</td>
</tr>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-6921. Allow resource request
to opt out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6921. Allow resource request to opt out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6fa51cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6fa51cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6fa51cb
Branch: refs/heads/YARN-1011
Commit: d6fa51cb1f0b37b7355a6cb773387c6a0b953450
Parents: dcb3709
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 09:03:05 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:28:47 2018 -0700
----------------------------------------------------------------------
.../scheduler/common/PendingAsk.java | 15 +++-
.../scheduler/fair/FSAppAttempt.java | 5 ++
.../LocalityAppPlacementAllocator.java | 27 ++++++-
.../scheduler/fair/FairSchedulerTestBase.java | 40 ++++++++--
.../scheduler/fair/TestFairScheduler.java | 77 ++++++++++++++++++++
5 files changed, 153 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6fa51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
index 2ed3e83..470dbbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
@@ -30,16 +30,21 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class PendingAsk {
private final Resource perAllocationResource;
private final int count;
- public final static PendingAsk ZERO = new PendingAsk(Resources.none(), 0);
+ public final static PendingAsk ZERO =
+ new PendingAsk(Resources.none(), 0, false);
+
+ private final boolean isGuaranteedTypeEnforced;
public PendingAsk(ResourceSizing sizing) {
this.perAllocationResource = sizing.getResources();
this.count = sizing.getNumAllocations();
+ this.isGuaranteedTypeEnforced = true;
}
- public PendingAsk(Resource res, int num) {
+ public PendingAsk(Resource res, int num, boolean guaranteedTypeEnforced) {
this.perAllocationResource = res;
this.count = num;
+ this.isGuaranteedTypeEnforced = guaranteedTypeEnforced;
}
public Resource getPerAllocationResource() {
@@ -50,11 +55,17 @@ public class PendingAsk {
return count;
}
+ public boolean isGuaranteedTypeEnforced() {
+ return isGuaranteedTypeEnforced;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("<per-allocation-resource=");
sb.append(getPerAllocationResource());
+ sb.append(", isGuaranteedEnforced=");
+ sb.append(isGuaranteedTypeEnforced());
sb.append(",repeat=");
sb.append(getCount());
sb.append(">");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6fa51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 1928591..0effd42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -859,6 +859,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
boolean reserved, boolean opportunistic,
SchedulerRequestKey schedulerKey) {
+ if (pendingAsk.isGuaranteedTypeEnforced() && opportunistic) {
+ // do not attempt to assign an OPPORTUNISTIC container to a resource
+ // request that has explicitly opted out of oversubscription
+ return Resources.none();
+ }
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6fa51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
index 4557350..68d53e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
@@ -188,13 +190,16 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
PendingAsk lastPendingAsk =
lastRequest == null ? null : new PendingAsk(
- lastRequest.getCapability(), lastRequest.getNumContainers());
+ lastRequest.getCapability(), lastRequest.getNumContainers(),
+ enforceGuaranteedExecutionType(lastRequest));
String lastRequestedNodePartition =
lastRequest == null ? null : lastRequest.getNodeLabelExpression();
updateResult = new PendingAskUpdateResult(lastPendingAsk,
new PendingAsk(request.getCapability(),
- request.getNumContainers()), lastRequestedNodePartition,
+ request.getNumContainers(),
+ enforceGuaranteedExecutionType(request)),
+ lastRequestedNodePartition,
request.getNodeLabelExpression());
}
}
@@ -234,8 +239,9 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
if (null == request) {
return PendingAsk.ZERO;
} else{
+ boolean guaranteedEnforced = enforceGuaranteedExecutionType(request);
return new PendingAsk(request.getCapability(),
- request.getNumContainers());
+ request.getNumContainers(), guaranteedEnforced);
}
} finally {
readLock.unlock();
@@ -243,6 +249,21 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
}
+ /**
+ * Check for a given ResourceRequest, if its guaranteed execution type
+ * needs to be enforced.
+ * @param request resource request
+ * @return true if its guaranteed execution type is to be enforced.
+ * false otherwise
+ */
+ private static boolean enforceGuaranteedExecutionType(
+ ResourceRequest request) {
+ ExecutionTypeRequest executionType = request.getExecutionTypeRequest();
+ return executionType != null &&
+ executionType.getExecutionType() == ExecutionType.GUARANTEED &&
+ executionType.getEnforceExecutionType();
+ }
+
@Override
public int getOutstandingAsksCount(String resourceName) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6fa51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 3ac3849..43a3931 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -117,9 +119,15 @@ public class FairSchedulerTestBase {
relaxLocality);
}
+ protected ResourceRequest createResourceRequest(int memory, int vcores,
+ String host, int priority, int numContainers, boolean relaxLocality) {
+ return createResourceRequest(memory, vcores, host, priority,
+ numContainers, relaxLocality, false);
+ }
+
protected ResourceRequest createResourceRequest(
int memory, int vcores, String host, int priority, int numContainers,
- boolean relaxLocality) {
+ boolean relaxLocality, boolean guaranteedExecutionEnforced) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
request.setCapability(BuilderUtils.newResource(memory, vcores));
request.setResourceName(host);
@@ -129,6 +137,11 @@ public class FairSchedulerTestBase {
request.setPriority(prio);
request.setRelaxLocality(relaxLocality);
request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ if (guaranteedExecutionEnforced) {
+ ExecutionTypeRequest executionType = ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true);
+ request.setExecutionTypeRequest(executionType);
+ }
return request;
}
@@ -152,6 +165,13 @@ public class FairSchedulerTestBase {
}
protected ApplicationAttemptId createSchedulingRequest(
+ int memory, String queueId, String userId,
+ int numContainers, boolean guaranteedExecutionEnforced) {
+ return createSchedulingRequest(memory, 1, queueId,
+ userId, numContainers, 1, guaranteedExecutionEnforced);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers) {
return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1);
}
@@ -165,16 +185,24 @@ public class FairSchedulerTestBase {
protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers,
int priority) {
- ResourceRequest request = createResourceRequest(memory, vcores,
- ResourceRequest.ANY, priority, numContainers, true);
+ return createSchedulingRequest(memory, vcores, queueId,
+ userId, numContainers, priority, false);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
+ int memory, int vcores, String queueId, String userId, int numContainers,
+ int priority, boolean guaranteedExecutionEnforced) {
+ ResourceRequest request = createResourceRequest(
+ memory, vcores, ResourceRequest.ANY, priority,
+ numContainers, true, guaranteedExecutionEnforced);
return createSchedulingRequest(Lists.newArrayList(request), queueId,
- userId);
+ userId);
}
protected ApplicationAttemptId createSchedulingRequest(
Collection<ResourceRequest> requests, String queueId, String userId) {
- ApplicationAttemptId id =
- createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
+ ApplicationAttemptId id = createAppAttemptId(this.APP_ID++,
+ this.ATTEMPT_ID++);
scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6fa51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ff620b8..fbb7243 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2712,6 +2712,83 @@ public class TestFairScheduler extends FairSchedulerTestBase {
getPriority().getPriority());
}
+ @Test
+ public void testResourceRequestOptOutOfOversubscription() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that opts out of oversubscription and
+ // asks for more than what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: Revert "YARN-6675. Add NM support
to launch opportunistic containers based on overallocation. Contributed by
Haibo Chen."
Posted by ha...@apache.org.
Revert "YARN-6675. Add NM support to launch opportunistic containers based on overallocation. Contributed by Haibo Chen."
This reverts commit c1362b68af03c546f4c0802758e9729d2372ab6c.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cf8e47a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cf8e47a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cf8e47a
Branch: refs/heads/YARN-1011
Commit: 4cf8e47abf2eb786bb360f9f585b5255cbfc0626
Parents: 08116cc
Author: Haibo Chen <ha...@apache.org>
Authored: Tue May 8 13:50:06 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:28:58 2018 -0700
----------------------------------------------------------------------
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 8 +-
.../launcher/ContainerLaunch.java | 2 +-
.../launcher/ContainersLauncher.java | 9 +-
.../monitor/ContainersMonitor.java | 38 +-
.../monitor/ContainersMonitorImpl.java | 56 +-
.../AllocationBasedResourceTracker.java | 114 --
...locationBasedResourceUtilizationTracker.java | 158 +++
.../scheduler/ContainerScheduler.java | 317 ++---
.../scheduler/ContainerSchedulerEventType.java | 4 +-
.../scheduler/NMAllocationPolicy.java | 63 -
.../scheduler/ResourceUtilizationTracker.java | 17 +-
.../SnapshotBasedOverAllocationPolicy.java | 54 -
.../UtilizationBasedResourceTracker.java | 95 --
.../BaseContainerManagerTest.java | 35 -
.../TestContainersMonitorResourceChange.java | 9 +-
.../TestAllocationBasedResourceTracker.java | 82 --
...locationBasedResourceUtilizationTracker.java | 93 ++
.../TestContainerSchedulerRecovery.java | 58 +-
...estContainerSchedulerWithOverAllocation.java | 1121 ------------------
20 files changed, 419 insertions(+), 1916 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index d757376..572684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -543,7 +543,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private ResourceUtilization getContainersUtilization() {
ContainersMonitor containersMonitor =
this.context.getContainerManager().getContainersMonitor();
- return containersMonitor.getContainersUtilization(false).getUtilization();
+ return containersMonitor.getContainersUtilization();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index a08e227..27a7c80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -245,12 +245,6 @@ public class ContainerManagerImpl extends CompositeService implements
metrics);
addService(rsrcLocalizationSrvc);
- this.containersMonitor = createContainersMonitor(exec);
- addService(this.containersMonitor);
-
- // ContainersLauncher must be added after ContainersMonitor
- // because the former depends on the latter to initialize
- // over-allocation first.
containersLauncher = createContainersLauncher(context, exec);
addService(containersLauncher);
@@ -275,6 +269,8 @@ public class ContainerManagerImpl extends CompositeService implements
nmMetricsPublisher = createNMTimelinePublisher(context);
context.setNMTimelinePublisher(nmMetricsPublisher);
}
+ this.containersMonitor = createContainersMonitor(exec);
+ addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
new ContainerEventDispatcher());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 0228332..6347d4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1079,7 +1079,7 @@ public class ContainerLaunch implements Callable<Integer> {
* @return Process ID
* @throws Exception
*/
- protected String getContainerPid(Path pidFilePath) throws Exception {
+ private String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 2f5acfa..7870f86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -121,7 +121,8 @@ public class ContainersLauncher extends AbstractService
containerId.getApplicationAttemptId().getApplicationId());
ContainerLaunch launch =
- createContainerLaunch(app, event.getContainer());
+ new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
+ event.getContainer(), dirsHandler, containerManager);
containerLauncher.submit(launch);
running.put(containerId, launch);
break;
@@ -224,10 +225,4 @@ public class ContainersLauncher extends AbstractService
break;
}
}
-
- protected ContainerLaunch createContainerLaunch(
- Application app, Container container) {
- return new ContainerLaunch(context, getConfig(), dispatcher,
- exec, app, container, dirsHandler, containerManager);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index 8da4ec4..64831e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -23,24 +23,10 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
public interface ContainersMonitor extends Service,
EventHandler<ContainersMonitorEvent>, ResourceView {
-
- /**
- * Get the aggregate resource utilization of containers running on the node,
- * with a timestamp of the measurement.
- * @param latest true if the latest result should be returned
- * @return ResourceUtilization resource utilization of all containers
- */
- ContainersResourceUtilization getContainersUtilization(boolean latest);
-
- /**
- * Get the policy to over-allocate containers when over-allocation is on.
- * @return null if over-allocation is turned off
- */
- NMAllocationPolicy getContainerOverAllocationPolicy();
+ ResourceUtilization getContainersUtilization();
float getVmemRatio();
@@ -80,26 +66,4 @@ public interface ContainersMonitor extends Service,
* containersMonitor.getVmemRatio());
resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores);
}
-
- /**
- * A snapshot of resource utilization of all containers with the timestamp.
- */
- final class ContainersResourceUtilization {
- private final ResourceUtilization utilization;
- private final long timestamp;
-
- public ContainersResourceUtilization(
- ResourceUtilization utilization, long timestamp) {
- this.utilization = utilization;
- this.timestamp = timestamp;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public ResourceUtilization getUtilization() {
- return utilization;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a045d78..7873882 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -25,10 +25,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,9 +112,8 @@ public class ContainersMonitorImpl extends AbstractService implements
CPU, MEMORY
}
- private ContainersResourceUtilization latestContainersUtilization;
+ private ResourceUtilization containersUtilization;
- private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
@@ -134,8 +129,7 @@ public class ContainersMonitorImpl extends AbstractService implements
this.monitoringThread = new MonitoringThread();
- this.latestContainersUtilization = new ContainersResourceUtilization(
- ResourceUtilization.newInstance(-1, -1, -1.0f), -1L);
+ this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
}
@Override
@@ -371,10 +365,6 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
cpuPreemptionThreshold, memoryPreemptionThreshold);
- // TODO make this configurable
- this.overAllocationPolicy =
- createOverAllocationPolicy(resourceThresholds);
-
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -382,11 +372,6 @@ public class ContainersMonitorImpl extends AbstractService implements
cpuPreemptionThreshold + ")");
}
- protected NMAllocationPolicy createOverAllocationPolicy(
- ResourceThresholds resourceThresholds) {
- return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
- }
-
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -670,12 +655,7 @@ public class ContainersMonitorImpl extends AbstractService implements
}
// Save the aggregated utilization of the containers
- setLatestContainersUtilization(trackedContainersUtilization);
-
- // check opportunity to start containers if over-allocation is on
- if (context.isOverAllocationEnabled()) {
- attemptToStartContainersUponLowUtilization();
- }
+ setContainersUtilization(trackedContainersUtilization);
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1065,34 +1045,12 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
- public ContainersResourceUtilization getContainersUtilization(
- boolean latest) {
- // TODO update containerUtilization if latest is true
- return this.latestContainersUtilization;
- }
-
- @Override
- public NMAllocationPolicy getContainerOverAllocationPolicy() {
- return overAllocationPolicy;
- }
-
- private void setLatestContainersUtilization(ResourceUtilization utilization) {
- this.latestContainersUtilization = new ContainersResourceUtilization(
- utilization, System.currentTimeMillis());
+ public ResourceUtilization getContainersUtilization() {
+ return this.containersUtilization;
}
- @VisibleForTesting
- public void attemptToStartContainersUponLowUtilization() {
- if (getContainerOverAllocationPolicy() != null) {
- Resource available = getContainerOverAllocationPolicy()
- .getAvailableResources();
- if (available.getMemorySize() > 0 &&
- available.getVirtualCores() > 0) {
- eventDispatcher.getEventHandler().handle(
- new ContainerSchedulerEvent(null,
- ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
- }
- }
+ private void setContainersUtilization(ResourceUtilization utilization) {
+ this.containersUtilization = utilization;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
deleted file mode 100644
index 86b3698..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An implementation of the resource utilization tracker that equates
- * resource utilization with the total resource allocated to the container.
- */
-public class AllocationBasedResourceTracker
- implements ResourceUtilizationTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
-
- private static final Resource UNAVAILABLE =
- Resource.newInstance(0, 0);
-
- private ResourceUtilization containersAllocation;
- private ContainerScheduler scheduler;
-
-
- AllocationBasedResourceTracker(ContainerScheduler scheduler) {
- this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
- this.scheduler = scheduler;
- }
-
- /**
- * Get the accumulation of totally allocated resources to containers.
- * @return ResourceUtilization Resource Utilization.
- */
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return this.containersAllocation;
- }
-
- /**
- * Get the amount of resources that have not been allocated to containers.
- * @return Resource resources that have not been allocated to containers.
- */
- protected Resource getUnallocatedResources() {
- // unallocated resources = node capacity - containers allocation
- // = -(container allocation - node capacity)
- ResourceUtilization allocationClone =
- ResourceUtilization.newInstance(containersAllocation);
- getContainersMonitor()
- .subtractNodeResourcesFromResourceUtilization(allocationClone);
-
- Resource unallocated = UNAVAILABLE;
- if (allocationClone.getCPU() <= 0 &&
- allocationClone.getPhysicalMemory() <= 0 &&
- allocationClone.getVirtualMemory() <= 0) {
- int cpu = Math.round(allocationClone.getCPU() *
- getContainersMonitor().getVCoresAllocatedForContainers());
- long memory = allocationClone.getPhysicalMemory();
- unallocated = Resource.newInstance(-memory, -cpu);
- }
- return unallocated;
- }
-
-
- @Override
- public Resource getAvailableResources() {
- return getUnallocatedResources();
- }
-
- /**
- * Add Container's resources to the accumulated allocation.
- * @param container Container.
- */
- @Override
- public void containerLaunched(Container container) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Subtract Container's resources to the accumulated allocation.
- * @param container Container.
- */
- @Override
- public void containerReleased(Container container) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- public ContainersMonitor getContainersMonitor() {
- return this.scheduler.getContainersMonitor();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
new file mode 100644
index 0000000..6e2b617
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the {@link ResourceUtilizationTracker} that equates
+ * resource utilization with the total resource allocated to the container.
+ */
+public class AllocationBasedResourceUtilizationTracker implements
+ ResourceUtilizationTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
+
+ private ResourceUtilization containersAllocation;
+ private ContainerScheduler scheduler;
+
+ AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) {
+ this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * Get the accumulation of totally allocated resources to a container.
+ * @return ResourceUtilization Resource Utilization.
+ */
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return this.containersAllocation;
+ }
+
+ /**
+ * Add Container's resources to the accumulated Utilization.
+ * @param container Container.
+ */
+ @Override
+ public void addContainerResources(Container container) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Subtract Container's resources to the accumulated Utilization.
+ * @param container Container.
+ */
+ @Override
+ public void subtractContainerResource(Container container) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Check if NM has resources available currently to run the container.
+ * @param container Container.
+ * @return True, if NM has resources available currently to run the container.
+ */
+ @Override
+ public boolean hasResourcesAvailable(Container container) {
+ long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
+ return hasResourcesAvailable(pMemBytes,
+ (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
+ container.getResource().getVirtualCores());
+ }
+
+ private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
+ int cpuVcores) {
+ // Check physical memory.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
+ this.containersAllocation.getPhysicalMemory(),
+ (pMemBytes >> 20),
+ (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
+ }
+ if (this.containersAllocation.getPhysicalMemory() +
+ (int) (pMemBytes >> 20) >
+ (int) (getContainersMonitor()
+ .getPmemAllocatedForContainers() >> 20)) {
+ return false;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("before vMemCheck" +
+ "[isEnabled={}, current={} + asked={} > allowed={}]",
+ getContainersMonitor().isVmemCheckEnabled(),
+ this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
+ (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
+ }
+ // Check virtual memory.
+ if (getContainersMonitor().isVmemCheckEnabled() &&
+ this.containersAllocation.getVirtualMemory() +
+ (int) (vMemBytes >> 20) >
+ (int) (getContainersMonitor()
+ .getVmemAllocatedForContainers() >> 20)) {
+ return false;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("before cpuCheck [asked={} > allowed={}]",
+ this.containersAllocation.getCPU(),
+ getContainersMonitor().getVCoresAllocatedForContainers());
+ }
+ // Check CPU. Compare using integral values of cores to avoid decimal
+ // inaccuracies.
+ if (!hasEnoughCpu(this.containersAllocation.getCPU(),
+ getContainersMonitor().getVCoresAllocatedForContainers(), cpuVcores)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns whether there is enough space for coresRequested in totalCores.
+ * Converts currentAllocation usage to nearest integer count before comparing,
+ * as floats are inherently imprecise. NOTE: this calculation assumes that
+ * requested core counts must be integers, and currentAllocation core count
+ * must also be an integer.
+ *
+ * @param currentAllocation The current allocation, a float value from 0 to 1.
+ * @param totalCores The total cores in the system.
+ * @param coresRequested The number of cores requested.
+ * @return True if currentAllocationtotalCores*coresRequested <=
+ * totalCores.
+ */
+ public boolean hasEnoughCpu(float currentAllocation, long totalCores,
+ int coresRequested) {
+ // Must not cast here, as it would truncate the decimal digits.
+ return Math.round(currentAllocation * totalCores)
+ + coresRequested <= totalCores;
+ }
+
+ public ContainersMonitor getContainersMonitor() {
+ return this.scheduler.getContainersMonitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0bebe44..a61b9d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -49,7 +48,6 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,10 +80,6 @@ public class ContainerScheduler extends AbstractService implements
// Queue of Guaranteed Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedGuaranteedContainers = new LinkedHashMap<>();
- // sum of the resources requested by guaranteed containers in queue
- private final Resource guaranteedResourcesDemanded =
- Resource.newInstance(0, 0);
-
// Queue of Opportunistic Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedOpportunisticContainers = new LinkedHashMap<>();
@@ -94,10 +88,6 @@ public class ContainerScheduler extends AbstractService implements
// or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
- // sum of the resources to be released by opportunistic containers that
- // have been marked to be killed or paused.
- private final Resource opportunisticResourcesToBeReleased =
- Resource.newInstance(0, 0);
// Containers launched by the Scheduler will take a while to actually
// move to the RUNNING state, but should still be fair game for killing
@@ -138,17 +128,6 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
- @VisibleForTesting
- public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
- NodeManagerMetrics metrics, int qLength) {
- super(ContainerScheduler.class.getName());
- this.context = context;
- this.dispatcher = dispatcher;
- this.metrics = metrics;
- this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
- this.opportunisticContainersStatus =
- OpportunisticContainersStatus.newInstance();
- }
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -176,16 +155,20 @@ public class ContainerScheduler extends AbstractService implements
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
YarnConfiguration.
DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
- // We assume over allocation configurations have been initialized
- this.utilizationTracker = getResourceTracker();
}
- private AllocationBasedResourceTracker getResourceTracker() {
- if (context.isOverAllocationEnabled()) {
- return new UtilizationBasedResourceTracker(this);
- } else {
- return new AllocationBasedResourceTracker(this);
- }
+ @VisibleForTesting
+ public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+ NodeManagerMetrics metrics, int qLength) {
+ super(ContainerScheduler.class.getName());
+ this.context = context;
+ this.dispatcher = dispatcher;
+ this.metrics = metrics;
+ this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
+ this.utilizationTracker =
+ new AllocationBasedResourceUtilizationTracker(this);
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
}
/**
@@ -208,18 +191,14 @@ public class ContainerScheduler extends AbstractService implements
if (event instanceof UpdateContainerSchedulerEvent) {
onUpdateContainer((UpdateContainerSchedulerEvent) event);
} else {
- LOG.error("Unknown event type on UpdateContainer: " + event.getType());
+ LOG.error("Unknown event type on UpdateCOntainer: " + event.getType());
}
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
case RECOVERY_COMPLETED:
- startPendingContainers(false);
- break;
- case SCHEDULE_CONTAINERS:
- startPendingContainers(true);
- break;
+ startPendingContainers(maxOppQueueLength <= 0);
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -234,10 +213,10 @@ public class ContainerScheduler extends AbstractService implements
ContainerId containerId = updateEvent.getContainer().getContainerId();
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
- this.utilizationTracker.containerReleased(
+ this.utilizationTracker.subtractContainerResource(
new ContainerImpl(getConfig(), null, null, null, null,
updateEvent.getOriginalToken(), context));
- this.utilizationTracker.containerLaunched(
+ this.utilizationTracker.addContainerResources(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
@@ -253,20 +232,17 @@ public class ContainerScheduler extends AbstractService implements
if (queuedOpportunisticContainers.remove(containerId) != null) {
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
- Resources.addTo(guaranteedResourcesDemanded,
- updateEvent.getContainer().getResource());
- startPendingContainers(true);
+ //Kill/pause opportunistic containers if any to make room for
+ // promotion request
+ reclaimOpportunisticContainerResources(updateEvent.getContainer());
}
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
// containers
if (queuedGuaranteedContainers.remove(containerId) != null) {
- Resources.subtractFrom(guaranteedResourcesDemanded,
- updateEvent.getContainer().getResource());
queuedOpportunisticContainers.put(containerId,
updateEvent.getContainer());
- startPendingContainers(false);
}
}
try {
@@ -293,7 +269,6 @@ public class ContainerScheduler extends AbstractService implements
|| rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else if (execType == ExecutionType.OPPORTUNISTIC) {
queuedOpportunisticContainers
.put(container.getContainerId(), container);
@@ -304,7 +279,7 @@ public class ContainerScheduler extends AbstractService implements
}
} else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
- utilizationTracker.containerLaunched(container);
+ utilizationTracker.addContainerResources(container);
}
if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
&& rcs.getCapability() != null) {
@@ -369,107 +344,65 @@ public class ContainerScheduler extends AbstractService implements
}
private void onResourcesReclaimed(Container container) {
- ContainerId containerId = container.getContainerId();
-
- // This could be killed externally for eg. by the ContainerManager,
- // in which case, the container might still be queued.
- if (queuedOpportunisticContainers.remove(containerId) != null) {
- return;
- }
+ oppContainersToKill.remove(container.getContainerId());
// This could be killed externally for eg. by the ContainerManager,
// in which case, the container might still be queued.
- if (queuedGuaranteedContainers.remove(containerId) != null) {
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
- return;
- }
-
- if (oppContainersToKill.remove(containerId) != null) {
- Resources.subtractFrom(
- opportunisticResourcesToBeReleased, container.getResource());
+ Container queued =
+ queuedOpportunisticContainers.remove(container.getContainerId());
+ if (queued == null) {
+ queuedGuaranteedContainers.remove(container.getContainerId());
}
// Requeue PAUSED containers
if (container.getContainerState() == ContainerState.PAUSED) {
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
- queuedGuaranteedContainers.put(containerId, container);
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
} else {
- queuedOpportunisticContainers.put(containerId, container);
+ queuedOpportunisticContainers.put(
+ container.getContainerId(), container);
}
}
// decrement only if it was a running container
- Container completedContainer = runningContainers.remove(containerId);
+ Container completedContainer = runningContainers.remove(container
+ .getContainerId());
// only a running container releases resources upon completion
boolean resourceReleased = completedContainer != null;
if (resourceReleased) {
- this.utilizationTracker.containerReleased(container);
+ this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
-
- // In case of over-allocation being turned on, we may need to reclaim
- // more resources since the opportunistic containers that have been
- // killed or paused may have not released as much resource as we need.
- boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
- startPendingContainers(reclaimOpportunisticResources);
+ boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
+ startPendingContainers(forceStartGuaranteedContainers);
}
}
/**
* Start pending containers in the queue.
- * @param reclaimOpportunisticResources if set to true, resources allocated
- * to running OPPORTUNISTIC containers will be reclaimed in
- * cases where there are GUARANTEED containers being queued
+ * @param forceStartGuaranteedContaieners When this is true, start guaranteed
+ * container without looking at available resource
*/
- private void startPendingContainers(boolean reclaimOpportunisticResources) {
- // When opportunistic container not allowed (which is determined by
- // max-queue length of pending opportunistic containers <= 0), start
- // guaranteed containers without looking at available resources and
- // skip scanning the queue of opportunistic containers
- if (maxOppQueueLength <= 0) {
- forcefullyStartGuaranteedContainers();
- return;
- }
-
- Resource available = utilizationTracker.getAvailableResources();
-
- // Start guaranteed containers that are queued, if resources available.
- boolean allGuaranteedContainersLaunched =
- startGuaranteedContainers(available);
- // Start opportunistic containers, if resources available, which is true
- // if all guaranteed containers in queue have been launched.
- if (allGuaranteedContainersLaunched) {
- startOpportunisticContainers(available);
- } else {
- // If not all guaranteed containers in queue are launched, we may need
- // to reclaim resources from opportunistic containers that are running.
- if (reclaimOpportunisticResources) {
- reclaimOpportunisticContainerResources();
- }
+ private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
+ // Start guaranteed containers that are paused, if resources available.
+ boolean resourcesAvailable = startContainers(
+ queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
+ // Start opportunistic containers, if resources available.
+ if (resourcesAvailable) {
+ startContainers(queuedOpportunisticContainers.values(), false);
}
}
- /**
- * Try to launch as many GUARANTEED containers as possible.
- * @param available the amount of resources available to launch containers
- * @return true if all queued GUARANTEED containers are launched
- * or there is no GUARANTEED containers to launch
- */
- private boolean startGuaranteedContainers(Resource available) {
- Iterator<Container> cIter =
- queuedGuaranteedContainers.values().iterator();
+ private boolean startContainers(
+ Collection<Container> containersToBeStarted, boolean force) {
+ Iterator<Container> cIter = containersToBeStarted.iterator();
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (isResourceAvailable(available, container)) {
- startContainer(container);
- Resources.subtractFrom(available, container.getResource());
+ if (tryStartContainer(container, force)) {
cIter.remove();
- Resources.subtractFrom(
- guaranteedResourcesDemanded, container.getResource());
} else {
resourcesAvailable = false;
}
@@ -477,49 +410,25 @@ public class ContainerScheduler extends AbstractService implements
return resourcesAvailable;
}
- /**
- * Launch all queued GUARANTEED containers without checking resource
- * availability. This is an optimization in cases where OPPORTUNISTIC
- * containers are not allowed on the node.
- */
- private void forcefullyStartGuaranteedContainers() {
- Iterator<Container> cIter =
- queuedGuaranteedContainers.values().iterator();
- while (cIter.hasNext()) {
- Container container = cIter.next();
+ private boolean tryStartContainer(Container container, boolean force) {
+ boolean containerStarted = false;
+ // call startContainer without checking available resource when force==true
+ if (force || resourceAvailableToStartContainer(
+ container)) {
startContainer(container);
- cIter.remove();
- Resources.subtractFrom(
- guaranteedResourcesDemanded, container.getResource());
+ containerStarted = true;
}
+ return containerStarted;
}
+
/**
- * Try to launch as many OPPORTUNISTIC containers as possible.
- * @param available the amount of resources available to launch containers
- * @return true if all OPPORTUNISTIC containers are launched
- * or there is no OPPORTUNISTIC containers to launch
+ * Check if there is resource available to start a given container
+ * immediately. (This can be extended to include overallocated resources)
+ * @param container the container to start
+ * @return true if container can be launched directly
*/
- private boolean startOpportunisticContainers(Resource available) {
- Iterator<Container> cIter =
- queuedOpportunisticContainers.values().iterator();
- boolean resourcesAvailable = true;
- while (cIter.hasNext() && resourcesAvailable) {
- Container container = cIter.next();
- if (isResourceAvailable(available, container)) {
- startContainer(container);
- Resources.subtractFrom(available, container.getResource());
- cIter.remove();
- } else {
- resourcesAvailable = false;
- }
- }
- return resourcesAvailable;
- }
-
- private static boolean isResourceAvailable(
- Resource resource, Container container) {
- Resource left = Resources.subtract(resource, container.getResource());
- return left.getMemorySize() >= 0 && left.getVirtualCores() >= 0;
+ private boolean resourceAvailableToStartContainer(Container container) {
+ return this.utilizationTracker.hasResourcesAvailable(container);
}
private boolean enqueueContainer(Container container) {
@@ -529,7 +438,6 @@ public class ContainerScheduler extends AbstractService implements
boolean isQueued;
if (isGuaranteedContainer) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
isQueued = true;
} else {
if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
@@ -574,7 +482,18 @@ public class ContainerScheduler extends AbstractService implements
// enough number of opportunistic containers.
if (isGuaranteedContainer) {
enqueueContainer(container);
- startPendingContainers(true);
+
+ // When opportunistic container not allowed (which is determined by
+ // max-queue length of pending opportunistic containers <= 0), start
+ // guaranteed containers without looking at available resources.
+ boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
+ startPendingContainers(forceStartGuaranteedContainers);
+
+ // if the guaranteed container is queued, we need to preempt opportunistic
+ // containers for make room for it
+ if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
+ reclaimOpportunisticContainerResources(container);
+ }
} else {
// Given an opportunistic container, we first try to start as many queuing
// guaranteed containers as possible followed by queuing opportunistic
@@ -592,19 +511,19 @@ public class ContainerScheduler extends AbstractService implements
}
@SuppressWarnings("unchecked")
- private void reclaimOpportunisticContainerResources() {
+ private void reclaimOpportunisticContainerResources(Container container) {
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources();
- killOpportunisticContainers(extraOppContainersToReclaim);
- }
-
- private void killOpportunisticContainers(
- Collection<Container> containersToReclaim) {
- for (Container contToReclaim : containersToReclaim) {
+ pickOpportunisticContainersToReclaimResources(
+ container.getContainerId());
+ // Kill the opportunistic containers that were chosen.
+ for (Container contToReclaim : extraOppContainersToReclaim) {
String preemptionAction = usePauseEventForPreemption == true ? "paused" :
- "preempted";
- LOG.info("Container {} will be {} to start the execution of guaranteed" +
- " containers.", contToReclaim.getContainerId(), preemptionAction);
+ "resumed";
+ LOG.info(
+ "Container {} will be {} to start the "
+ + "execution of guaranteed container {}.",
+ contToReclaim.getContainerId(), preemptionAction,
+ container.getContainerId());
if (usePauseEventForPreemption) {
contToReclaim.sendPauseEvent(
@@ -615,8 +534,6 @@ public class ContainerScheduler extends AbstractService implements
"Container Killed to make room for Guaranteed Container.");
}
oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
- Resources.addTo(
- opportunisticResourcesToBeReleased, contToReclaim.getResource());
}
}
@@ -625,7 +542,7 @@ public class ContainerScheduler extends AbstractService implements
// Skip to put into runningContainers and addUtilization when recover
if (!runningContainers.containsKey(container.getContainerId())) {
runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.containerLaunched(container);
+ this.utilizationTracker.addContainerResources(container);
}
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -634,12 +551,14 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources() {
+ private List<Container> pickOpportunisticContainersToReclaimResources(
+ ContainerId containerToStartId) {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp(
+ containerToStartId);
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
@@ -658,19 +577,15 @@ public class ContainerScheduler extends AbstractService implements
continue;
}
extraOpportContainersToKill.add(runningCont);
- // In the case of over-allocation, the running container may not
- // release as much resources as it has requested, but we'll check
- // again if more containers need to be killed/paused when this
- // container is released.
ContainersMonitor.decreaseResourceUtilization(
getContainersMonitor(), resourcesToFreeUp,
runningCont.getResource());
}
}
if (!hasSufficientResources(resourcesToFreeUp)) {
- LOG.warn("There are no sufficient resources to start guaranteed" +
- " containers at the moment. Opportunistic containers are in" +
- " the process of being killed to make room.");
+ LOG.warn("There are no sufficient resources to start guaranteed [{}]" +
+ "at the moment. Opportunistic containers are in the process of" +
+ "being killed to make room.", containerToStartId);
}
return extraOpportContainersToKill;
}
@@ -685,42 +600,34 @@ public class ContainerScheduler extends AbstractService implements
* getContainersMonitor().getVCoresAllocatedForContainers()) <= 0;
}
- /**
- * Determine how much resources are needed to be freed up to launch the given
- * GUARANTEED container. Used to determine how many running OPPORTUNISTIC
- * containers need to be killed/paused, assuming OPPORTUNISTIC containers to
- * be killed/paused will release the amount of resources they have requested.
- *
- * If the node is over-allocating itself, this may cause not enough
- * OPPORTUNISTIC containers being killed/paused in cases where the running
- * OPPORTUNISTIC containers are not consuming fully their resource requests.
- * We'd check again upon container completion events to see if more running
- * OPPORTUNISTIC containers need to be killed/paused.
- *
- * @return the amount of resource needed to be reclaimed for this container
- */
- private ResourceUtilization resourcesToFreeUp() {
+ private ResourceUtilization resourcesToFreeUp(
+ ContainerId containerToStartId) {
// Get allocation of currently allocated containers.
ResourceUtilization resourceAllocationToFreeUp = ResourceUtilization
- .newInstance(0, 0, 0.0f);
-
- // Add to the allocation the allocation of pending guaranteed containers.
- ContainersMonitor.increaseResourceUtilization(getContainersMonitor(),
- resourceAllocationToFreeUp, guaranteedResourcesDemanded);
+ .newInstance(this.utilizationTracker.getCurrentUtilization());
+
+ // Add to the allocation the allocation of the pending guaranteed
+ // containers that will start before the current container will be started.
+ for (Container container : queuedGuaranteedContainers.values()) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), resourceAllocationToFreeUp,
+ container.getResource());
+ if (container.getContainerId().equals(containerToStartId)) {
+ break;
+ }
+ }
// These resources are being freed, likely at the behest of another
// guaranteed container..
- ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
- resourceAllocationToFreeUp, opportunisticResourcesToBeReleased);
-
- // Deduct any remaining resources available
- Resource availableResources = utilizationTracker.getAvailableResources();
- if (availableResources.getVirtualCores() > 0 &&
- availableResources.getMemorySize() > 0) {
- ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
- resourceAllocationToFreeUp, availableResources);
+ for (Container container : oppContainersToKill.values()) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), resourceAllocationToFreeUp,
+ container.getResource());
}
+ // Subtract the overall node resources.
+ getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
+ resourceAllocationToFreeUp);
return resourceAllocationToFreeUp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 9ad4f91..294eddf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -28,7 +28,5 @@ public enum ContainerSchedulerEventType {
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
CONTAINER_PAUSED,
- RECOVERY_COMPLETED,
- // Producer: Containers Monitor when over-allocation is on
- SCHEDULE_CONTAINERS
+ RECOVERY_COMPLETED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
deleted file mode 100644
index 58b73d2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-
-/**
- * Keeps track of containers utilization over time and determines how much
- * resources are available to launch containers when over-allocation is on.
- */
-public abstract class NMAllocationPolicy {
- protected final ResourceThresholds overAllocationThresholds;
- protected final ContainersMonitor containersMonitor;
-
- public NMAllocationPolicy(
- ResourceThresholds overAllocationThresholds,
- ContainersMonitor containersMonitor) {
- this.containersMonitor = containersMonitor;
- this.overAllocationThresholds = overAllocationThresholds;
- }
-
- /**
- * Handle container launch events.
- * @param container the container that has been launched
- */
- public void containerLaunched(Container container) {
-
- }
-
- /**
- * Handle container release events.
- * @param container the container that has been released
- */
- public void containerReleased(Container container) {
-
- }
-
- /**
- * Get the amount of resources to launch containers when
- * over-allocation is turned on.
- * @return the amount of resources available to launch containers
- */
- public abstract Resource getAvailableResources();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 98d99c6..3c17eca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -39,20 +38,22 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
- * Get the amount of resources currently available to launch containers.
- * @return Resource resources available to launch containers
+ * Add Container's resources to Node Utilization.
+ * @param container Container.
*/
- Resource getAvailableResources();
+ void addContainerResources(Container container);
/**
- * Add Container's resources to Node Utilization upon container launch.
+ * Subtract Container's resources to Node Utilization.
* @param container Container.
*/
- void containerLaunched(Container container);
+ void subtractContainerResource(Container container);
/**
- * Subtract Container's resources to Node Utilization upon container release.
+ * Check if NM has resources available currently to run the container.
* @param container Container.
+ * @return True, if NM has resources available currently to run the container.
*/
- void containerReleased(Container container);
+ boolean hasResourcesAvailable(Container container);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
deleted file mode 100644
index f486506..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-
-/**
- * An implementation of NMAllocationPolicy based on the
- * snapshot of the latest containers utilization to determine how much
- * resources are available * to launch containers when over-allocation
- * is turned on.
- */
-public class SnapshotBasedOverAllocationPolicy
- extends NMAllocationPolicy {
-
- public SnapshotBasedOverAllocationPolicy(
- ResourceThresholds overAllocationThresholds,
- ContainersMonitor containersMonitor) {
- super(overAllocationThresholds, containersMonitor);
- }
-
- @Override
- public Resource getAvailableResources() {
- ResourceUtilization utilization =
- containersMonitor.getContainersUtilization(true).getUtilization();
- long memoryAvailable = Math.round(
- overAllocationThresholds.getMemoryThreshold() *
- containersMonitor.getPmemAllocatedForContainers()) -
- (utilization.getPhysicalMemory() << 20);
- int vcoreAvailable = Math.round(
- (overAllocationThresholds.getCpuThreshold() - utilization.getCPU()) *
- containersMonitor.getVCoresAllocatedForContainers());
- return Resource.newInstance(memoryAvailable >> 20, vcoreAvailable);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
deleted file mode 100644
index 6f9bc82..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
-* An resource availability tracker that determines if there are resources
-* available based on if there are unallocated resources or if there are
-* un-utilized resources.
-*/
-public class UtilizationBasedResourceTracker
- extends AllocationBasedResourceTracker {
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
-
- private final NMAllocationPolicy overAllocationPolicy;
-
- UtilizationBasedResourceTracker(ContainerScheduler scheduler) {
- super(scheduler);
- this.overAllocationPolicy =
- getContainersMonitor().getContainerOverAllocationPolicy();
- }
-
- @Override
- public void containerLaunched(Container container) {
- super.containerLaunched(container);
- if (overAllocationPolicy != null) {
- overAllocationPolicy.containerLaunched(container);
- }
- }
-
- @Override
- public void containerReleased(Container container) {
- super.containerReleased(container);
- if (overAllocationPolicy != null) {
- overAllocationPolicy.containerReleased(container);
- }
- }
-
- @Override
- public Resource getAvailableResources() {
- Resource resourceBasedOnAllocation = getUnallocatedResources();
- Resource resourceBasedOnUtilization =
- getResourcesAvailableBasedOnUtilization();
- if (LOG.isDebugEnabled()) {
- LOG.debug("The amount of resources available based on allocation is " +
- resourceBasedOnAllocation + ", based on utilization is " +
- resourceBasedOnUtilization);
- }
-
- return Resources.componentwiseMax(resourceBasedOnAllocation,
- resourceBasedOnUtilization);
- }
-
- /**
- * Get the amount of resources based on the slack between
- * the actual utilization and desired utilization.
- * @return Resource resource available
- */
- private Resource getResourcesAvailableBasedOnUtilization() {
- if (overAllocationPolicy == null) {
- return Resources.none();
- }
-
- return overAllocationPolicy.getAvailableResources();
- }
-
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return getContainersMonitor().getContainersUtilization(false)
- .getUtilization();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 92613ed..493aa4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,7 +26,6 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
-import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -346,40 +345,6 @@ public abstract class BaseContainerManagerTest {
fStates.contains(containerStatus.getState()));
}
- public static void waitForContainerSubState(
- ContainerManagementProtocol containerManager, ContainerId containerID,
- ContainerSubState finalState)
- throws InterruptedException, YarnException, IOException {
- waitForContainerSubState(containerManager, containerID,
- Arrays.asList(finalState), 20);
- }
- public static void waitForContainerSubState(
- ContainerManagementProtocol containerManager, ContainerId containerID,
- List<ContainerSubState> finalStates, int timeOutMax)
- throws InterruptedException, YarnException, IOException {
- List<ContainerId> list = new ArrayList<>();
- list.add(containerID);
- GetContainerStatusesRequest request =
- GetContainerStatusesRequest.newInstance(list);
- ContainerStatus containerStatus;
- HashSet<ContainerSubState> fStates = new HashSet<>(finalStates);
- int timeoutSecs = 0;
- do {
- Thread.sleep(1000);
- containerStatus =
- containerManager.getContainerStatuses(request)
- .getContainerStatuses().get(0);
- LOG.info("Waiting for container to get into one of states " + fStates
- + ". Current state is " + containerStatus.getContainerSubState());
- timeoutSecs += 1;
- } while (!fStates.contains(containerStatus.getContainerSubState())
- && timeoutSecs < timeOutMax);
- LOG.info("Container state is " + containerStatus.getContainerSubState());
- Assert.assertTrue("ContainerSubState is not correct (timedout)",
- fStates.contains(containerStatus.getContainerSubState()));
- }
-
-
public static void waitForApplicationState(
ContainerManagerImpl containerManager, ApplicationId appID,
ApplicationState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index c071283..8aee532 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -288,7 +288,7 @@ public class TestContainersMonitorResourceChange {
// will be 0.
assertEquals(
"Resource utilization must be default with MonitorThread's first run",
- 0, containersMonitor.getContainersUtilization(false).getUtilization()
+ 0, containersMonitor.getContainersUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
// Verify the container utilization value. Since atleast one round is done,
@@ -303,9 +303,8 @@ public class TestContainersMonitorResourceChange {
ContainersMonitorImpl containersMonitor, int timeoutMsecs)
throws InterruptedException {
int timeWaiting = 0;
- while (0 == containersMonitor.getContainersUtilization(false)
- .getUtilization().compareTo(
- ResourceUtilization.newInstance(0, 0, 0.0f))) {
+ while (0 == containersMonitor.getContainersUtilization()
+ .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) {
if (timeWaiting >= timeoutMsecs) {
break;
}
@@ -317,7 +316,7 @@ public class TestContainersMonitorResourceChange {
}
assertTrue("Resource utilization is not changed from second run onwards",
- 0 != containersMonitor.getContainersUtilization(false).getUtilization()
+ 0 != containersMonitor.getContainersUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
deleted file mode 100644
index 1e8bfdf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for the {@link AllocationBasedResourceTracker} class.
- */
-public class TestAllocationBasedResourceTracker {
-
- private ContainerScheduler mockContainerScheduler;
-
- @Before
- public void setup() {
- mockContainerScheduler = mock(ContainerScheduler.class);
- ContainersMonitor containersMonitor =
- new ContainersMonitorImpl(mock(ContainerExecutor.class),
- mock(AsyncDispatcher.class), mock(Context.class));
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
- conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
- conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
- conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
- conf.setInt(YarnConfiguration.NM_VCORES, 8);
- containersMonitor.init(conf);
- when(mockContainerScheduler.getContainersMonitor())
- .thenReturn(containersMonitor);
- }
-
- /**
- * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
- * hasResourceAvailable should return false.
- */
- @Test
- public void testHasResourcesAvailable() {
- AllocationBasedResourceTracker tracker =
- new AllocationBasedResourceTracker(mockContainerScheduler);
- Container testContainer = mock(Container.class);
- when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
- for (int i = 0; i < 2; i++) {
- Assert.assertTrue(
- isResourcesAvailable(tracker.getAvailableResources(), testContainer));
- tracker.containerLaunched(testContainer);
- }
- Assert.assertFalse(
- isResourcesAvailable(tracker.getAvailableResources(), testContainer));
- }
-
- private static boolean isResourcesAvailable(
- Resource available, Container container) {
- return available.compareTo(container.getResource()) >= 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cf8e47a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
new file mode 100644
index 0000000..82c2147
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link AllocationBasedResourceUtilizationTracker} class.
+ */
+public class TestAllocationBasedResourceUtilizationTracker {
+
+ private ContainerScheduler mockContainerScheduler;
+
+ @Before
+ public void setup() {
+ mockContainerScheduler = mock(ContainerScheduler.class);
+ ContainersMonitor containersMonitor =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
+ conf.setInt(YarnConfiguration.NM_VCORES, 8);
+ containersMonitor.init(conf);
+ when(mockContainerScheduler.getContainersMonitor())
+ .thenReturn(containersMonitor);
+ }
+
+ /**
+ * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
+ * hasResourceAvailable should return false.
+ */
+ @Test
+ public void testHasResourcesAvailable() {
+ AllocationBasedResourceUtilizationTracker tracker =
+ new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
+ Container testContainer = mock(Container.class);
+ when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
+ for (int i = 0; i < 2; i++) {
+ Assert.assertTrue(tracker.hasResourcesAvailable(testContainer));
+ tracker.addContainerResources(testContainer);
+ }
+ Assert.assertFalse(tracker.hasResourcesAvailable(testContainer));
+ }
+
+ /**
+ * Test the case where the current allocation has been truncated to 0.8888891
+ * (8/9 cores used). Request 1 additional core - hasEnoughCpu should return
+ * true.
+ */
+ @Test
+ public void testHasEnoughCpu() {
+ AllocationBasedResourceUtilizationTracker tracker =
+ new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
+ float currentAllocation = 0.8888891f;
+ long totalCores = 9;
+ int alreadyUsedCores = 8;
+ Assert.assertTrue(tracker.hasEnoughCpu(currentAllocation, totalCores,
+ (int) totalCores - alreadyUsedCores));
+ Assert.assertFalse(tracker.hasEnoughCpu(currentAllocation, totalCores,
+ (int) totalCores - alreadyUsedCores + 1));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-8784.
DockerLinuxContainerRuntime prevents access to distributed cache entries on a
full disk. Contributed by Eric Badger
Posted by ha...@apache.org.
YARN-8784. DockerLinuxContainerRuntime prevents access to distributed cache entries on a full disk. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b5838ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b5838ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b5838ed
Branch: refs/heads/YARN-1011
Commit: 6b5838ed3220f992092c7348f92f1d9d0d4a3061
Parents: f6bb1ca
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 19 16:44:51 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 19 16:44:51 2018 -0500
----------------------------------------------------------------------
.../launcher/ContainerLaunch.java | 5 +-
.../launcher/TestContainerLaunch.java | 81 ++++++++++++++++++++
2 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5838ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 2aca5f8..6347d4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -250,12 +250,13 @@ public class ContainerLaunch implements Callable<Integer> {
// accessible by users
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
List<String> localDirs = dirsHandler.getLocalDirs();
+ List<String> localDirsForRead = dirsHandler.getLocalDirsForRead();
List<String> logDirs = dirsHandler.getLogDirs();
- List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
+ List<String> filecacheDirs = getNMFilecacheDirs(localDirsForRead);
List<String> userLocalDirs = getUserLocalDirs(localDirs);
List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
List<String> containerLogDirs = getContainerLogDirs(logDirs);
- List<String> userFilecacheDirs = getUserFilecacheDirs(localDirs);
+ List<String> userFilecacheDirs = getUserFilecacheDirs(localDirsForRead);
List<String> applicationLocalDirs = getApplicationLocalDirs(localDirs,
appIdStr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5838ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index ddf46a6..5714a1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -107,6 +108,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -121,6 +123,7 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -2362,4 +2365,82 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
}
}
+
+ @Test
+ public void testDistributedCacheDirs() throws Exception {
+ Container container = mock(Container.class);
+ ApplicationId appId =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ ContainerId containerId = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1);
+ when(container.getContainerId()).thenReturn(containerId);
+ when(container.getUser()).thenReturn("test");
+
+ when(container.getLocalizedResources())
+ .thenReturn(Collections.<Path, List<String>> emptyMap());
+ Dispatcher dispatcher = mock(Dispatcher.class);
+
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
+ when(container.getLaunchContext()).thenReturn(clc);
+
+ @SuppressWarnings("rawtypes")
+ ContainerExitHandler eventHandler =
+ mock(ContainerExitHandler.class);
+ when(dispatcher.getEventHandler()).thenReturn(eventHandler);
+
+ Application app = mock(Application.class);
+ when(app.getAppId()).thenReturn(appId);
+ when(app.getUser()).thenReturn("test");
+
+ Credentials creds = mock(Credentials.class);
+ when(container.getCredentials()).thenReturn(creds);
+
+ ((NMContext) context).setNodeId(NodeId.newInstance("127.0.0.1", HTTP_PORT));
+ ContainerExecutor mockExecutor = mock(ContainerExecutor.class);
+
+ LocalDirsHandlerService mockDirsHandler =
+ mock(LocalDirsHandlerService.class);
+
+ List <String> localDirsForRead = new ArrayList<String>();
+ String localDir1 =
+ new File("target", this.getClass().getSimpleName() + "-localDir1")
+ .getAbsoluteFile().toString();
+ String localDir2 =
+ new File("target", this.getClass().getSimpleName() + "-localDir2")
+ .getAbsoluteFile().toString();
+ localDirsForRead.add(localDir1);
+ localDirsForRead.add(localDir2);
+
+ List <String> localDirs = new ArrayList();
+ localDirs.add(localDir1);
+ Path logPathForWrite = new Path(localDirs.get(0));
+
+ when(mockDirsHandler.areDisksHealthy()).thenReturn(true);
+ when(mockDirsHandler.getLocalDirsForRead()).thenReturn(localDirsForRead);
+ when(mockDirsHandler.getLocalDirs()).thenReturn(localDirs);
+ when(mockDirsHandler.getLogDirs()).thenReturn(localDirs);
+ when(mockDirsHandler.getLogPathForWrite(anyString(),
+ anyBoolean())).thenReturn(logPathForWrite);
+ when(mockDirsHandler.getLocalPathForWrite(anyString()))
+ .thenReturn(logPathForWrite);
+ when(mockDirsHandler.getLocalPathForWrite(anyString(), anyLong(),
+ anyBoolean())).thenReturn(logPathForWrite);
+
+ ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher,
+ mockExecutor, app, container, mockDirsHandler, containerManager);
+ launch.call();
+
+ ArgumentCaptor <ContainerStartContext> ctxCaptor =
+ ArgumentCaptor.forClass(ContainerStartContext.class);
+ verify(mockExecutor, times(1)).launchContainer(ctxCaptor.capture());
+ ContainerStartContext ctx = ctxCaptor.getValue();
+
+ Assert.assertEquals(StringUtils.join(",",
+ launch.getNMFilecacheDirs(localDirsForRead)),
+ StringUtils.join(",", ctx.getFilecacheDirs()));
+ Assert.assertEquals(StringUtils.join(",",
+ launch.getUserFilecacheDirs(localDirsForRead)),
+ StringUtils.join(",", ctx.getUserFilecacheDirs()));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-507. EventQueue should be
shutdown on SCM shutdown. Contributed by Xiaoyu Yao.
Posted by ha...@apache.org.
HDDS-507. EventQueue should be shutdown on SCM shutdown. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/236d16e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/236d16e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/236d16e3
Branch: refs/heads/YARN-1011
Commit: 236d16e3a5611b8ca4f80cb7e8ffd63f167db8e0
Parents: 042bf74
Author: Nanda kumar <na...@apache.org>
Authored: Thu Sep 20 01:22:18 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Thu Sep 20 01:22:18 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/server/StorageContainerManager.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/236d16e3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 86c061b..b2cbc93 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -759,6 +759,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
unregisterMXBean();
+ // Event queue must be stopped before the DB store is closed at the end.
+ try {
+ LOG.info("Stopping SCM Event Queue.");
+ eventQueue.close();
+ } catch (Exception ex) {
+ LOG.error("SCM Event Queue stop failed", ex);
+ }
IOUtils.cleanupWithLogger(LOG, scmContainerManager);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node
resource utilization and allocate opportunistic containers if appropriate.
Posted by ha...@apache.org.
YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/495d43ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/495d43ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/495d43ad
Branch: refs/heads/YARN-1011
Commit: 495d43adb76aba3122ad9064f56c7358a0893894
Parents: ce4c4a7
Author: Haibo Chen <ha...@apache.org>
Authored: Fri Nov 17 07:47:32 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:03:25 2018 -0700
----------------------------------------------------------------------
.../sls/scheduler/FairSchedulerMetrics.java | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 13 +
.../scheduler/SchedulerNode.java | 48 ++
.../scheduler/fair/FSAppAttempt.java | 175 ++++---
.../scheduler/fair/FSLeafQueue.java | 50 +-
.../scheduler/fair/FSParentQueue.java | 36 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 39 +-
.../scheduler/fair/FairScheduler.java | 105 +++--
.../fair/FairSchedulerConfiguration.java | 5 +
.../scheduler/fair/Schedulable.java | 17 +-
.../DominantResourceFairnessPolicy.java | 8 +-
.../fair/policies/FairSharePolicy.java | 4 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 60 ++-
.../TestWorkPreservingRMRestart.java | 2 +-
.../scheduler/fair/FakeSchedulable.java | 9 +-
.../scheduler/fair/TestAppRunnability.java | 9 +-
.../scheduler/fair/TestFSAppAttempt.java | 4 +-
.../scheduler/fair/TestFSLeafQueue.java | 4 +-
.../scheduler/fair/TestFSSchedulerNode.java | 4 +-
.../scheduler/fair/TestFairScheduler.java | 468 +++++++++++++++++--
.../scheduler/fair/TestSchedulingPolicy.java | 10 +-
23 files changed, 868 insertions(+), 213 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index a5aee74..1f4e7c7 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -75,7 +75,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
case DEMAND:
return schedulable.getDemand().getMemorySize();
case USAGE:
- return schedulable.getResourceUsage().getMemorySize();
+ return schedulable.getGuaranteedResourceUsage().getMemorySize();
case MINSHARE:
return schedulable.getMinShare().getMemorySize();
case MAXSHARE:
@@ -96,7 +96,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
case DEMAND:
return schedulable.getDemand().getVirtualCores();
case USAGE:
- return schedulable.getResourceUsage().getVirtualCores();
+ return schedulable.getGuaranteedResourceUsage().getVirtualCores();
case MINSHARE:
return schedulable.getMinShare().getVirtualCores();
case MAXSHARE:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 18ff086..71651a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -278,6 +278,11 @@ public class YarnConfiguration extends Configuration {
public static final String APP_NAME_PLACEMENT_RULE = "app-name";
+ public static final String RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED =
+ RM_PREFIX + "scheduler.oversubscription.enabled";
+ public static final boolean DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED
+ = false;
+
/** Enable Resource Manager webapp ui actions */
public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =
RM_PREFIX + "webapp.ui-actions.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 985936a..b45fe06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -429,6 +429,19 @@
</property>
<property>
+ <description>
+ If set to true, the scheduler will try to over-allocate resources on the
+ nodes that allow overallocation. To enable overallocatin on a node, set
+ {code}yarn.nodemanager.overallocation.memory-utilization-threshold{code}
+ and
+ {code}yarn.nodemanager.overallocation.cpu-utilization-threshold{code}
+ to a number in the range (0.0, 1.0)
+ </description>
+ <name>yarn.resourcemanager.scheduler.oversubscription.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified. </description>
<name>yarn.resourcemanager.recovery.enabled</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 8f5ae0e..ff3640c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -62,6 +64,10 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
private Resource capacity;
+ // The resource available within the node's capacity that can be given out
+ // to run GUARANTEED containers, including reserved, preempted and any
+ // remaining free resources. Resources allocated to OPPORTUNISTIC containers
+ // are tracked in allocatedResourceOpportunistic
private Resource unallocatedResource = Resource.newInstance(0, 0);
private RMContainer reservedContainer;
@@ -663,6 +669,48 @@ public abstract class SchedulerNode {
this.nodeAttributes = attributes;
}
+ /**
+ * Get the amount of resources that can be allocated to opportunistic
+ * containers in the case of overallocation. It is calculated as
+ * node capacity - (node utilization + resources of allocated-yet-not-started
+ * containers).
+ * @return the amount of resources that are available to be allocated to
+ * opportunistic containers
+ */
+ public synchronized Resource allowedResourceForOverAllocation() {
+ OverAllocationInfo overAllocationInfo = rmNode.getOverAllocationInfo();
+ if (overAllocationInfo == null) {
+ LOG.debug("Overallocation is disabled on node: " + rmNode.getHostName());
+ return Resources.none();
+ }
+
+ ResourceUtilization projectedNodeUtilization = ResourceUtilization.
+ newInstance(getNodeUtilization());
+ // account for resources allocated in this heartbeat
+ projectedNodeUtilization.addTo(
+ (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0,
+ (float) resourceAllocatedPendingLaunch.getVirtualCores() /
+ capacity.getVirtualCores());
+
+ ResourceThresholds thresholds =
+ overAllocationInfo.getOverAllocationThresholds();
+ Resource overAllocationThreshold = Resources.createResource(
+ (long) (capacity.getMemorySize() * thresholds.getMemoryThreshold()),
+ (int) (capacity.getVirtualCores() * thresholds.getCpuThreshold()));
+ long allowedMemory = Math.max(0, overAllocationThreshold.getMemorySize()
+ - projectedNodeUtilization.getPhysicalMemory());
+ int allowedCpu = Math.max(0, (int)
+ (overAllocationThreshold.getVirtualCores() -
+ projectedNodeUtilization.getCPU() * capacity.getVirtualCores()));
+
+ Resource resourceAllowedForOpportunisticContainers =
+ Resources.createResource(allowedMemory, allowedCpu);
+
+ // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
+ // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ return resourceAllowedForOpportunisticContainers;
+ }
+
private static class ContainerInfo {
private final RMContainer container;
private boolean launchedOnNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index d9f3262..1928591 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -169,7 +170,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedResource(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
@@ -178,30 +179,35 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
}
- private void unreserveInternal(
+ private boolean unreserveInternal(
SchedulerRequestKey schedulerKey, FSSchedulerNode node) {
try {
writeLock.lock();
Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(
schedulerKey);
- RMContainer reservedContainer = reservedContainers.remove(
- node.getNodeID());
- if (reservedContainers.isEmpty()) {
- this.reservedContainers.remove(schedulerKey);
- }
+ boolean unreserved = false;
+ if (reservedContainers != null) {
+ RMContainer reservedContainer = reservedContainers.remove(
+ node.getNodeID());
+ if (reservedContainers.isEmpty()) {
+ this.reservedContainers.remove(schedulerKey);
+ }
- // Reset the re-reservation count
- resetReReservations(schedulerKey);
+ // Reset the re-reservation count
+ resetReReservations(schedulerKey);
- Resource resource = reservedContainer.getContainer().getResource();
- this.attemptResourceUsage.decReserved(resource);
+ Resource resource = reservedContainer.getContainer().getResource();
+ this.attemptResourceUsage.decReserved(resource);
+ unreserved = true;
- LOG.info(
- "Application " + getApplicationId() + " unreserved " + " on node "
- + node + ", currently has " + reservedContainers.size()
- + " at priority " + schedulerKey.getPriority()
- + "; currentReservation " + this.attemptResourceUsage
- .getReserved());
+ LOG.info(
+ "Application " + getApplicationId() + " unreserved " + " on node "
+ + node + ", currently has " + reservedContainers.size()
+ + " at priority " + schedulerKey.getPriority()
+ + "; currentReservation " + this.attemptResourceUsage
+ .getReserved());
+ }
+ return unreserved;
} finally {
writeLock.unlock();
}
@@ -229,7 +235,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
SchedulingPolicy policy = fsQueue.getPolicy();
Resource queueFairShare = fsQueue.getFairShare();
- Resource queueUsage = fsQueue.getResourceUsage();
+ Resource queueUsage = fsQueue.getGuaranteedResourceUsage();
Resource clusterResource = this.scheduler.getClusterResource();
Resource clusterUsage = this.scheduler.getRootQueueMetrics()
.getAllocatedResources();
@@ -420,7 +426,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public RMContainer allocate(NodeType type, FSSchedulerNode node,
SchedulerRequestKey schedulerKey, PendingAsk pendingAsk,
- Container reservedContainer) {
+ Container reservedContainer, boolean opportunistic) {
RMContainer rmContainer;
Container container;
@@ -445,9 +451,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
container = reservedContainer;
+ ExecutionType executionType = opportunistic ?
+ ExecutionType.OPPORTUNISTIC : ExecutionType.GUARANTEED;
if (container == null) {
container = createContainer(node, pendingAsk.getPerAllocationResource(),
- schedulerKey);
+ schedulerKey, executionType);
}
// Create RMContainer
@@ -462,8 +470,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Update consumption and track allocations
ContainerRequest containerRequest = appSchedulingInfo.allocate(
type, node, schedulerKey, container);
- this.attemptResourceUsage.incUsed(container.getResource());
- getQueue().incUsedResource(container.getResource());
+ if (executionType.equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.incUsed(container.getResource());
+ getQueue().incUsedGuaranteedResource(container.getResource());
+ } else {
+ this.attemptOpportunisticResourceUsage.incUsed(container.getResource());
+ }
// Update resource requests related to "request" and store in RMContainer
((RMContainerImpl) rmContainer).setContainerRequest(containerRequest);
@@ -620,7 +632,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private Resource getUsageAfterPreemptingContainer(Resource containerResources,
Resource alreadyConsideringForPreemption) {
- Resource usageAfterPreemption = Resources.clone(getResourceUsage());
+ Resource usageAfterPreemption =
+ Resources.clone(getGuaranteedResourceUsage());
// Subtract resources of containers already queued for preemption
synchronized (preemptionVariablesLock) {
@@ -647,7 +660,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return Container
*/
private Container createContainer(FSSchedulerNode node, Resource capability,
- SchedulerRequestKey schedulerKey) {
+ SchedulerRequestKey schedulerKey, ExecutionType executionType) {
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(
@@ -657,7 +670,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return BuilderUtils.newContainer(containerId, nodeId,
node.getRMNode().getHttpAddress(), capability,
schedulerKey.getPriority(), null,
- schedulerKey.getAllocationRequestId());
+ executionType, schedulerKey.getAllocationRequestId());
}
@Override
@@ -669,7 +682,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
super.recoverContainer(node, rmContainer);
if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
- getQueue().incUsedResource(rmContainer.getContainer().getResource());
+ getQueue().incUsedGuaranteedResource(
+ rmContainer.getContainer().getResource());
}
// If not running unmanaged, the first container we recover is always
@@ -707,7 +721,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (reservedContainer == null) {
reservedContainer =
createContainer(node, perAllocationResource,
- schedulerKey);
+ schedulerKey, ExecutionType.GUARANTEED);
getMetrics().reserveResource(node.getPartition(), getUser(),
reservedContainer.getResource());
RMContainer rmContainer =
@@ -764,11 +778,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public void unreserve(SchedulerRequestKey schedulerKey,
FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
- unreserveInternal(schedulerKey, node);
- node.unreserveResource(this);
- clearReservation(node);
- getMetrics().unreserveResource(node.getPartition(),
- getUser(), rmContainer.getContainer().getResource());
+ if (unreserveInternal(schedulerKey, node)) {
+ node.unreserveResource(this);
+ clearReservation(node);
+ getMetrics().unreserveResource(node.getPartition(),
+ getUser(), rmContainer.getContainer().getResource());
+ }
}
private void setReservation(SchedulerNode node) {
@@ -842,13 +857,15 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*/
private Resource assignContainer(
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
- boolean reserved, SchedulerRequestKey schedulerKey) {
+ boolean reserved, boolean opportunistic,
+ SchedulerRequestKey schedulerKey) {
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
// How much does the node have?
- Resource available = node.getUnallocatedResource();
+ Resource available = opportunistic ?
+ node.allowedResourceForOverAllocation() : node.getUnallocatedResource();
Container reservedContainer = null;
if (reserved) {
@@ -860,39 +877,46 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Inform the application of the new container for this request
RMContainer allocatedContainer =
allocate(type, node, schedulerKey, pendingAsk,
- reservedContainer);
- if (allocatedContainer == null) {
- // Did the application need this resource?
- if (reserved) {
+ reservedContainer, opportunistic);
+
+ // delete the previous reservation, if any
+ if (reserved) {
+ unreserve(schedulerKey, node);
+ }
+
+ if (allocatedContainer != null) {
+ if (opportunistic) {
+ // if an OPPORTUNISTIC container is allocated, we need to
+ // unreserve anything that we may have reserved in our
+ // previous attempt to assign GUARANTEED containers for this
+ // scheduling request.
unreserve(schedulerKey, node);
}
+
+
+ // Inform the node
+ node.allocateContainer(allocatedContainer);
+
+ // If not running unmanaged, the first container we allocate
+ // is always the AM. Set amResource for this app and update
+ // the leaf queue's AM usage
+ if (!isAmRunning() && !getUnmanagedAM()) {
+ setAMResource(capability);
+ getQueue().addAMResourceUsage(capability);
+ setAmRunning(true);
+ }
+
+ return capability;
+ } else {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format(
"Resource ask %s fits in available node resources %s, " +
- "but no container was allocated",
+ "but no container was allocated",
capability, available));
}
- return Resources.none();
- }
- // If we had previously made a reservation, delete it
- if (reserved) {
- unreserve(schedulerKey, node);
- }
-
- // Inform the node
- node.allocateContainer(allocatedContainer);
-
- // If not running unmanaged, the first container we allocate is always
- // the AM. Set the amResource for this app and update the leaf queue's AM
- // usage
- if (!isAmRunning() && !getUnmanagedAM()) {
- setAMResource(capability);
- getQueue().addAMResourceUsage(capability);
- setAmRunning(true);
+ return Resources.none();
}
-
- return capability;
}
if (LOG.isDebugEnabled()) {
@@ -903,7 +927,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// The desired container won't fit here, so reserve
// Reserve only, if app does not wait for preempted resources on the node,
// otherwise we may end up with duplicate reservations
- if (isReservable(capability) &&
+ if (isReservable(capability) && !opportunistic &&
!node.isPreemptedForApp(this) &&
reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
type, schedulerKey)) {
@@ -949,7 +973,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@SuppressWarnings("deprecation")
- private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
+ private Resource assignContainer(FSSchedulerNode node, boolean opportunistic,
+ boolean reserved) {
if (LOG.isTraceEnabled()) {
LOG.trace("Node offered to app: " + getName() + " reserved: " + reserved);
}
@@ -1012,7 +1037,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, nodeLocalPendingAsk, NodeType.NODE_LOCAL,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
if (!appSchedulingInfo.canDelayTo(schedulerKey, node.getRackName())) {
@@ -1029,7 +1054,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, rackLocalPendingAsk, NodeType.RACK_LOCAL,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
PendingAsk offswitchAsk = getPendingAsk(schedulerKey,
@@ -1049,7 +1074,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, offswitchAsk, NodeType.OFF_SWITCH,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
}
@@ -1150,7 +1175,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// there's only one container size per priority.
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
node.getUnallocatedResource())) {
- assignContainer(node, true);
+ assignContainer(node, false, true);
}
return true;
}
@@ -1166,7 +1191,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource fairDemand = Resources.componentwiseMin(threshold, demand);
// Check if the queue is starved for fairshare
- boolean starved = isUsageBelowShare(getResourceUsage(), fairDemand);
+ boolean starved =
+ isUsageBelowShare(getGuaranteedResourceUsage(), fairDemand);
if (!starved) {
lastTimeAtFairShare = now;
@@ -1178,8 +1204,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
fairshareStarvation = Resources.none();
} else {
// The app has been starved for longer than preemption-timeout.
- fairshareStarvation =
- Resources.subtractFromNonNegative(fairDemand, getResourceUsage());
+ fairshareStarvation = Resources.subtractFromNonNegative(fairDemand,
+ getGuaranteedResourceUsage());
}
return fairshareStarvation;
}
@@ -1198,7 +1224,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return true if the app is starved for fairshare, false otherwise
*/
boolean isStarvedForFairShare() {
- return isUsageBelowShare(getResourceUsage(), getFairShare());
+ return isUsageBelowShare(getGuaranteedResourceUsage(), getFairShare());
}
/**
@@ -1299,7 +1325,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* Get the current app's unsatisfied demand.
*/
Resource getPendingDemand() {
- return Resources.subtract(demand, getResourceUsage());
+ return Resources.subtract(demand, getGuaranteedResourceUsage());
}
@Override
@@ -1318,11 +1344,16 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return getCurrentConsumption();
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return attemptOpportunisticResourceUsage.getUsed();
+ }
+
+ @Override
public float getWeight() {
float weight = 1.0F;
@@ -1371,7 +1402,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
if (isOverAMShareLimit()) {
PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
updateAMDiagnosticMsg(amAsk.getPerAllocationResource(),
@@ -1383,7 +1414,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
return Resources.none();
}
- return assignContainer(node, false);
+ return assignContainer(node, opportunistic, false);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index cbc74d2..3834130 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -97,7 +97,7 @@ public class FSLeafQueue extends FSQueue {
// when an appAttempt is created for an application, we'd like to move
// it over from assignedApps to either runnableApps or nonRunnableApps
assignedApps.remove(app.getApplicationId());
- incUsedResource(app.getResourceUsage());
+ incUsedGuaranteedResource(app.getGuaranteedResourceUsage());
} finally {
writeLock.unlock();
}
@@ -132,7 +132,7 @@ public class FSLeafQueue extends FSQueue {
getMetrics().setAMResourceUsage(amResourceUsage);
}
- decUsedResource(app.getResourceUsage());
+ decUsedGuaranteedResource(app.getGuaranteedResourceUsage());
return runnable;
}
@@ -302,6 +302,42 @@ public class FSLeafQueue extends FSQueue {
return demand;
}
+ @Override
+ public Resource getGuaranteedResourceUsage() {
+ Resource guaranteedResource = Resources.createResource(0);
+ readLock.lock();
+ try {
+ for (FSAppAttempt app : runnableApps) {
+ Resources.addTo(guaranteedResource, app.getGuaranteedResourceUsage());
+ }
+ for (FSAppAttempt app : nonRunnableApps) {
+ Resources.addTo(guaranteedResource, app.getGuaranteedResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return guaranteedResource;
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsage() {
+ Resource opportunisticResource = Resource.newInstance(0, 0);
+ readLock.lock();
+ try {
+ for (FSAppAttempt app : runnableApps) {
+ Resources.addTo(opportunisticResource,
+ app.getOpportunisticResourceUsage());
+ }
+ for (FSAppAttempt app : nonRunnableApps) {
+ Resources.addTo(opportunisticResource,
+ app.getOpportunisticResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return opportunisticResource;
+ }
+
Resource getAmResourceUsage() {
return amResourceUsage;
}
@@ -335,14 +371,14 @@ public class FSLeafQueue extends FSQueue {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
Resource assigned = none();
if (LOG.isDebugEnabled()) {
LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
getName() + " fairShare: " + getFairShare());
}
- if (!assignContainerPreCheck(node)) {
+ if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
@@ -350,7 +386,7 @@ public class FSLeafQueue extends FSQueue {
if (SchedulerAppUtils.isPlaceBlacklisted(sched, node, LOG)) {
continue;
}
- assigned = sched.assignContainer(node);
+ assigned = sched.assignContainer(node, opportunistic);
if (!assigned.equals(none())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigned container in queue:" + getName() + " " +
@@ -557,7 +593,7 @@ public class FSLeafQueue extends FSQueue {
Resource starvation =
Resources.componentwiseMin(getMinShare(), getDemand());
- Resources.subtractFromNonNegative(starvation, getResourceUsage());
+ Resources.subtractFromNonNegative(starvation, getGuaranteedResourceUsage());
boolean starved = !Resources.isNone(starvation);
long now = scheduler.getClock().getTime();
@@ -616,7 +652,7 @@ public class FSLeafQueue extends FSQueue {
", SteadyFairShare: " + getSteadyFairShare() +
", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
- ", ResourceUsage: " + getResourceUsage() +
+ ", ResourceUsage: " + getGuaranteedResourceUsage() +
", Demand: " + getDemand() +
", Runnable: " + getNumRunnableApps() +
", NumPendingApps: " + getNumPendingApps() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index d5df549..17a74c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -119,6 +119,34 @@ public class FSParentQueue extends FSQueue {
}
@Override
+ public Resource getGuaranteedResourceUsage() {
+ Resource guaranteedResource = Resources.createResource(0);
+ readLock.lock();
+ try {
+ for (FSQueue child : childQueues) {
+ Resources.addTo(guaranteedResource, child.getGuaranteedResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return guaranteedResource;
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsage() {
+ Resource opportunisticResource = Resource.newInstance(0, 0);
+ readLock.lock();
+ try {
+ for (FSQueue child : childQueues) {
+ Resources.addTo(opportunisticResource,
+ child.getOpportunisticResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return opportunisticResource;
+ }
+
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
@@ -177,11 +205,11 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
Resource assigned = Resources.none();
// If this queue is over its limit, reject
- if (!assignContainerPreCheck(node)) {
+ if (!assignContainerPreCheck(node, opportunistic)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assign container precheck on node " + node + " failed");
}
@@ -201,7 +229,7 @@ public class FSParentQueue extends FSQueue {
try {
sortedChildQueues.addAll(childQueues);
for (FSQueue child : sortedChildQueues) {
- assigned = child.assignContainer(node);
+ assigned = child.assignContainer(node, opportunistic);
if (!Resources.equals(assigned, Resources.none())) {
break;
}
@@ -285,7 +313,7 @@ public class FSParentQueue extends FSQueue {
", SteadyFairShare: " + getSteadyFairShare() +
", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
- ", ResourceUsage: " + getResourceUsage() +
+ ", Guaranteed ResourceUsage: " + getGuaranteedResourceUsage() +
", Demand: " + getDemand() +
", MaxAMShare: " + maxAMShare +
", Runnable: " + getNumRunnableApps() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 6b88a32..52c76f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -57,7 +57,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private Resource fairShare = Resources.createResource(0, 0);
private Resource steadyFairShare = Resources.createResource(0, 0);
private Resource reservedResource = Resources.createResource(0, 0);
- private final Resource resourceUsage = Resource.newInstance(0, 0);
+ private final Resource guaranteedResourceUsage = Resource.newInstance(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final YarnAuthorizationProvider authorizer;
@@ -235,7 +235,8 @@ public abstract class FSQueue implements Queue, Schedulable {
if (getFairShare().getMemorySize() == 0) {
queueInfo.setCurrentCapacity(0.0f);
} else {
- queueInfo.setCurrentCapacity((float) getResourceUsage().getMemorySize() /
+ queueInfo.setCurrentCapacity(
+ (float) getGuaranteedResourceUsage().getMemorySize() /
getFairShare().getMemorySize());
}
@@ -419,14 +420,17 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @return true if check passes (can assign) or false otherwise
*/
- boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (node.getReservedContainer() != null) {
+ boolean assignContainerPreCheck(FSSchedulerNode node, boolean opportunistic) {
+ if (opportunistic) {
+ // always pre-approve OPPORTUNISTIC containers to be assigned on the node
+ return true;
+ } else if (node.getReservedContainer() != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because it has reserved containers.");
}
return false;
- } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) {
+ } else if (!Resources.fitsIn(getGuaranteedResourceUsage(), getMaxShare())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because queue resource usage is larger than MaxShare: "
@@ -449,7 +453,8 @@ public abstract class FSQueue implements Queue, Schedulable {
@Override
public String toString() {
return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]",
- getName(), getDemand(), getResourceUsage(), fairShare, getWeight());
+ getName(), getDemand(), getGuaranteedResourceUsage(), fairShare,
+ getWeight());
}
@Override
@@ -481,8 +486,8 @@ public abstract class FSQueue implements Queue, Schedulable {
}
@Override
- public Resource getResourceUsage() {
- return resourceUsage;
+ public Resource getGuaranteedResourceUsage() {
+ return guaranteedResourceUsage;
}
/**
@@ -490,11 +495,11 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param res the resource to increase
*/
- protected void incUsedResource(Resource res) {
- synchronized (resourceUsage) {
- Resources.addTo(resourceUsage, res);
+ protected void incUsedGuaranteedResource(Resource res) {
+ synchronized (guaranteedResourceUsage) {
+ Resources.addTo(guaranteedResourceUsage, res);
if (parent != null) {
- parent.incUsedResource(res);
+ parent.incUsedGuaranteedResource(res);
}
}
}
@@ -504,11 +509,11 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param res the resource to decrease
*/
- protected void decUsedResource(Resource res) {
- synchronized (resourceUsage) {
- Resources.subtractFrom(resourceUsage, res);
+ protected void decUsedGuaranteedResource(Resource res) {
+ synchronized (guaranteedResourceUsage) {
+ Resources.subtractFrom(guaranteedResourceUsage, res);
if (parent != null) {
- parent.decUsedResource(res);
+ parent.decUsedGuaranteedResource(res);
}
}
}
@@ -521,7 +526,7 @@ public abstract class FSQueue implements Queue, Schedulable {
boolean fitsInMaxShare(Resource additionalResource) {
Resource usagePlusAddition =
- Resources.add(getResourceUsage(), additionalResource);
+ Resources.add(getGuaranteedResourceUsage(), additionalResource);
if (!Resources.fitsIn(usagePlusAddition, getMaxShare())) {
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 43a47ae..5d48fac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -173,7 +173,6 @@ public class FairScheduler extends
private float reservableNodesRatio; // percentage of available nodes
// an app can be reserved on
-
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
// Continuous Scheduling enabled or not
@Deprecated
@@ -196,6 +195,8 @@ public class FairScheduler extends
boolean maxAssignDynamic;
protected int maxAssign; // Max containers to assign per heartbeat
+ protected boolean oversubscriptionEnabled;
+
@VisibleForTesting
final MaxRunningAppsEnforcer maxRunningEnforcer;
@@ -1075,13 +1076,13 @@ public class FairScheduler extends
* resources for preempted containers.
* @param node Node to check
*/
- static void assignPreemptedContainers(FSSchedulerNode node) {
+ static void attemptToAssignPreemptedResources(FSSchedulerNode node) {
for (Entry<FSAppAttempt, Resource> entry :
node.getPreemptionList().entrySet()) {
FSAppAttempt app = entry.getKey();
Resource preemptionPending = Resources.clone(entry.getValue());
while (!app.isStopped() && !Resources.isNone(preemptionPending)) {
- Resource assigned = app.assignContainer(node);
+ Resource assigned = app.assignContainer(node, false);
if (Resources.isNone(assigned) ||
assigned.equals(FairScheduler.CONTAINER_RESERVED)) {
// Fail to assign, let's not try further
@@ -1113,49 +1114,85 @@ public class FairScheduler extends
// Assign new containers...
// 1. Ensure containers are assigned to the apps that preempted
// 2. Check for reserved applications
- // 3. Schedule if there are no reservations
+ // 3. Schedule GUARANTEED containers if there are no reservations
+ // 4. Schedule OPPORTUNISTIC containers if possible
// Apps may wait for preempted containers
// We have to satisfy these first to avoid cases, when we preempt
// a container for A from B and C gets the preempted containers,
// when C does not qualify for preemption itself.
- assignPreemptedContainers(node);
- FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
- boolean validReservation = false;
- if (reservedAppSchedulable != null) {
- validReservation = reservedAppSchedulable.assignReservedContainer(node);
- }
+ attemptToAssignPreemptedResources(node);
+
+ boolean validReservation = attemptToAssignReservedResources(node);
if (!validReservation) {
- // No reservation, schedule at queue which is farthest below fair share
- int assignedContainers = 0;
- Resource assignedResource = Resources.clone(Resources.none());
- Resource maxResourcesToAssign = Resources.multiply(
- node.getUnallocatedResource(), 0.5f);
-
- while (node.getReservedContainer() == null) {
- Resource assignment = queueMgr.getRootQueue().assignContainer(node);
-
- if (assignment.equals(Resources.none())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No container is allocated on node " + node);
- }
- break;
- }
+ // only attempt to assign GUARANTEED containers if there is no
+ // reservation on the node because
+ attemptToAssignResourcesAsGuaranteedContainers(node);
+ }
- assignedContainers++;
- Resources.addTo(assignedResource, assignment);
- if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign,
- assignedResource)) {
- break;
- }
- }
+ // attempt to assign OPPORTUNISTIC containers regardless of whether
+ // we have made a reservation or assigned a GUARANTEED container
+ if (oversubscriptionEnabled) {
+ attemptToAssignResourcesAsOpportunisticContainers(node);
}
+
updateRootQueueMetrics();
} finally {
writeLock.unlock();
}
}
+ /**
+ * Assign the reserved resource to the application that have reserved it.
+ */
+ private boolean attemptToAssignReservedResources(FSSchedulerNode node) {
+ boolean success = false;
+ FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
+ if (reservedAppSchedulable != null) {
+ success = reservedAppSchedulable.assignReservedContainer(node);
+ }
+ return success;
+ }
+
+ private void attemptToAssignResourcesAsGuaranteedContainers(
+ FSSchedulerNode node) {
+ // No reservation, schedule at queue which is farthest below fair share
+ int assignedContainers = 0;
+ Resource assignedResource = Resources.clone(Resources.none());
+ Resource maxResourcesToAssign = Resources.multiply(
+ node.getUnallocatedResource(), 0.5f);
+ while (node.getReservedContainer() == null) {
+ Resource assignment =
+ queueMgr.getRootQueue().assignContainer(node, false);
+ if (assignment.equals(Resources.none())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No container is allocated on node " + node);
+ }
+ break;
+ }
+ assignedContainers++;
+ Resources.addTo(assignedResource, assignment);
+
+ if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign,
+ assignedResource)) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Try to assign OPPORTUNISTIC containers as long as there is resources
+ * to.
+ * @param node the node to assign OPPORTUNISTIC containers on
+ */
+ private void attemptToAssignResourcesAsOpportunisticContainers(
+ FSSchedulerNode node) {
+ while (!Resources.none().equals(
+ queueMgr.getRootQueue().assignContainer(node, true))) {
+ // nothing to do here
+ }
+ }
+
public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) {
return super.getApplicationAttempt(appAttemptId);
}
@@ -1398,6 +1435,7 @@ public class FairScheduler extends
sizeBasedWeight = this.conf.getSizeBasedWeight();
usePortForNodeName = this.conf.getUsePortForNodeName();
reservableNodesRatio = this.conf.getReservableNodes();
+ oversubscriptionEnabled = this.conf.isOversubscriptionEnabled();
updateInterval = this.conf.getUpdateInterval();
if (updateInterval < 0) {
@@ -1801,7 +1839,8 @@ public class FairScheduler extends
}
// maxShare
- if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption),
+ if (!Resources.fitsIn(
+ Resources.add(cur.getGuaranteedResourceUsage(), consumption),
cur.getMaxShare())) {
throw new YarnException("Moving app attempt " + appAttId + " to queue "
+ queueName + " would violate queue maxShare constraints on"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 8c4932b..48f91ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -348,6 +348,11 @@ public class FairSchedulerConfiguration extends Configuration {
DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS);
}
+ public boolean isOversubscriptionEnabled() {
+ return getBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED);
+ }
+
/**
* Delay in milliseconds for locality fallback node to rack.
* @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
index bd1ff7a..f018929 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
@@ -58,8 +58,17 @@ public interface Schedulable {
*/
Resource getDemand();
- /** Get the aggregate amount of resources consumed by the schedulable. */
- Resource getResourceUsage();
+ /**
+ * Get the aggregate amount of guaranteed resources consumed by the
+ * schedulable.
+ */
+ Resource getGuaranteedResourceUsage();
+
+ /**
+ * Get the aggregate amount of opportunistic resources consumed by the
+ * schedulable.
+ */
+ Resource getOpportunisticResourceUsage();
/** Minimum Resource share assigned to the schedulable. */
Resource getMinShare();
@@ -89,8 +98,10 @@ public interface Schedulable {
/**
* Assign a container on this node if possible, and return the amount of
* resources assigned.
+ * @param node the node to assign containers on
+ * @param opportunistic whether to assign OPPORTUNISTIC containers or not
*/
- Resource assignContainer(FSSchedulerNode node);
+ Resource assignContainer(FSSchedulerNode node, boolean opportunistic);
/** Get the fair share assigned to this Schedulable. */
Resource getFairShare();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 59635d9..f366891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -169,8 +169,8 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
extends DominantResourceFairnessComparator {
@Override
public int compare(Schedulable s1, Schedulable s2) {
- Resource usage1 = s1.getResourceUsage();
- Resource usage2 = s2.getResourceUsage();
+ Resource usage1 = s1.getGuaranteedResourceUsage();
+ Resource usage2 = s2.getGuaranteedResourceUsage();
Resource minShare1 = s1.getMinShare();
Resource minShare2 = s2.getMinShare();
Resource clusterCapacity = fsContext.getClusterResource();
@@ -370,9 +370,9 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public int compare(Schedulable s1, Schedulable s2) {
ResourceInformation[] resourceInfo1 =
- s1.getResourceUsage().getResources();
+ s1.getGuaranteedResourceUsage().getResources();
ResourceInformation[] resourceInfo2 =
- s2.getResourceUsage().getResources();
+ s2.getGuaranteedResourceUsage().getResources();
ResourceInformation[] minShareInfo1 = s1.getMinShare().getResources();
ResourceInformation[] minShareInfo2 = s2.getMinShare().getResources();
ResourceInformation[] clusterInfo =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 0b5c10b..bce9c35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -93,8 +93,8 @@ public class FairSharePolicy extends SchedulingPolicy {
Resource resourceUsage2 = null;
if (res == 0) {
- resourceUsage1 = s1.getResourceUsage();
- resourceUsage2 = s2.getResourceUsage();
+ resourceUsage1 = s1.getGuaranteedResourceUsage();
+ resourceUsage2 = s2.getGuaranteedResourceUsage();
res = compareMinShareUsage(s1, s2, resourceUsage1, resourceUsage2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 913513c..fe313a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -88,7 +88,7 @@ public class FairSchedulerQueueInfo {
amMaxResources = new ResourceInfo(Resource.newInstance(
queue.getMetrics().getMaxAMShareMB(),
queue.getMetrics().getMaxAMShareVCores()));
- usedResources = new ResourceInfo(queue.getResourceUsage());
+ usedResources = new ResourceInfo(queue.getGuaranteedResourceUsage());
demandResources = new ResourceInfo(queue.getDemand());
fractionMemUsed = (float)usedResources.getMemorySize() /
clusterResources.getMemorySize();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index ce1d6b5..bff6125 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -105,7 +104,10 @@ public class MockNodes {
return rs;
}
- private static class MockRMNodeImpl implements RMNode {
+ /**
+ * A mock implementation of RMNode.
+ */
+ public static class MockRMNodeImpl implements RMNode {
private NodeId nodeId;
private String hostName;
private String nodeAddr;
@@ -120,12 +122,27 @@ public class MockNodes {
private ResourceUtilization containersUtilization;
private ResourceUtilization nodeUtilization;
private Resource physicalResource;
+ private OverAllocationInfo overAllocationInfo;
+ private List<UpdatedContainerInfo> containerUpdates =
+ Collections.EMPTY_LIST;
+
+ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
+ Resource perNode, String rackName, String healthReport,
+ long lastHealthReportTime, int cmdPort, String hostName,
+ NodeState state, Set<String> labels,
+ ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
+ this(nodeId, nodeAddr, httpAddress, perNode, rackName, healthReport,
+ lastHealthReportTime, cmdPort, hostName, state, labels,
+ containersUtilization, nodeUtilization, pPhysicalResource, null);
+ }
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
Set<String> labels, ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource,
+ OverAllocationInfo overAllocationInfo) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress;
@@ -140,6 +157,7 @@ public class MockNodes {
this.containersUtilization = containersUtilization;
this.nodeUtilization = nodeUtilization;
this.physicalResource = pPhysicalResource;
+ this.overAllocationInfo = overAllocationInfo;
}
@Override
@@ -228,7 +246,7 @@ public class MockNodes {
@Override
public List<UpdatedContainerInfo> pullContainerUpdates() {
- return new ArrayList<UpdatedContainerInfo>();
+ return containerUpdates;
}
@Override
@@ -266,7 +284,7 @@ public class MockNodes {
@Override
public OverAllocationInfo getOverAllocationInfo() {
- return null;
+ return this.overAllocationInfo;
}
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
@@ -311,6 +329,19 @@ public class MockNodes {
public Resource getPhysicalResource() {
return this.physicalResource;
}
+
+ public void updateResourceUtilization(ResourceUtilization utilization) {
+ this.nodeUtilization = utilization;
+ }
+
+ public void updateContainersAndNodeUtilization(
+ UpdatedContainerInfo updatedContainerInfo,
+ ResourceUtilization resourceUtilization) {
+ if (updatedContainerInfo != null) {
+ containerUpdates = Collections.singletonList(updatedContainerInfo);
+ }
+ this.nodeUtilization = resourceUtilization;
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
@@ -334,6 +365,15 @@ public class MockNodes {
NodeState state, String httpAddr, int hostnum, String hostName, int port,
Set<String> labels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Resource physicalResource) {
+ return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
+ labels, containersUtilization, nodeUtilization, physicalResource, null);
+ }
+
+ private static MockRMNodeImpl buildRMNode(int rack, final Resource perNode,
+ NodeState state, String httpAddr, int hostnum, String hostName, int port,
+ Set<String> labels, ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization, Resource physicalResource,
+ OverAllocationInfo overAllocationInfo) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
@@ -346,9 +386,9 @@ public class MockNodes {
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
rackName, healthReport, 0, nid, hostName, state, labels,
- containersUtilization, nodeUtilization, physicalResource);
+ containersUtilization, nodeUtilization,
+ physicalResource, overAllocationInfo);
}
-
public static RMNode nodeInfo(int rack, final Resource perNode,
NodeState state) {
return buildRMNode(rack, perNode, state, "N/A");
@@ -377,4 +417,10 @@ public class MockNodes {
return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0", hostnum, hostName, port);
}
+ public static MockRMNodeImpl newNodeInfo(int rack, final Resource perNode,
+ OverAllocationInfo overAllocationInfo) {
+ return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0",
+ NODE_ID++, null, 123, null, ResourceUtilization.newInstance(0, 0, 0.0f),
+ ResourceUtilization.newInstance(0, 0, 0.0f), null, overAllocationInfo);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index df409f5..077c01a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -508,7 +508,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
FSParentQueue root = scheduler.getQueueManager().getRootQueue();
// ************ check cluster used Resources ********
assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
- assertEquals(usedResources,root.getResourceUsage());
+ assertEquals(usedResources, root.getGuaranteedResourceUsage());
// ************ check app headroom ****************
FSAppAttempt schedulerAttempt =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index b1fc2d0..c1d628b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
@@ -84,7 +84,7 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
return null;
}
@@ -114,11 +114,16 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return usage;
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
public long getStartTime() {
return startTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index f581935..71db786 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -222,7 +222,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
scheduler.handle(nodeEvent);
scheduler.handle(updateEvent);
- assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1),
+ oldQueue.getGuaranteedResourceUsage());
scheduler.update();
assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
@@ -231,8 +232,10 @@ public class TestAppRunnability extends FairSchedulerTestBase {
assertSame(targetQueue, app.getQueue());
assertFalse(oldQueue.isRunnableApp(app));
assertTrue(targetQueue.isRunnableApp(app));
- assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
- assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(0, 0),
+ oldQueue.getGuaranteedResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1),
+ targetQueue.getGuaranteedResourceUsage());
assertEquals(0, oldQueue.getNumRunnableApps());
assertEquals(1, targetQueue.getNumRunnableApps());
assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
index 51ffd23..9bfbb91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
@@ -223,7 +223,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
- Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
+ Mockito.when(mockQueue.getGuaranteedResourceUsage()).thenReturn(queueUsage);
Mockito.when(mockScheduler.getClusterResource()).thenReturn
(clusterResource);
Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
@@ -305,7 +305,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
getApplicationId()));
FSAppAttempt app = scheduler.getSchedulerApp(id11);
assertNotNull(app);
- Resource queueUsage = app.getQueue().getResourceUsage();
+ Resource queueUsage = app.getQueue().getGuaranteedResourceUsage();
assertEquals(0, queueUsage.getMemorySize());
assertEquals(0, queueUsage.getVirtualCores());
SchedulerNode n1 = scheduler.getSchedulerNode(node1.getNodeID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 4a738ca..efe4ad1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -88,7 +88,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(maxResource);
- Mockito.when(app.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app.getGuaranteedResourceUsage()).thenReturn(Resources.none());
schedulable.addApp(app, true);
schedulable.addApp(app, true);
@@ -176,7 +176,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
@Override
public void run() {
for (int i=0; i < 500; i++) {
- schedulable.getResourceUsage();
+ schedulable.getGuaranteedResourceUsage();
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 6726f17..f79ba4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -98,7 +98,7 @@ public class TestFSSchedulerNode {
ApplicationAttemptId appAttemptId =
mock(ApplicationAttemptId.class);
when(starvingApp.getApplicationAttemptId()).thenReturn(appAttemptId);
- when(starvingApp.assignContainer(schedulerNode)).thenAnswer(
+ when(starvingApp.assignContainer(schedulerNode, false)).thenAnswer(
new Answer<Resource>() {
@Override
public Resource answer(InvocationOnMock invocationOnMock)
@@ -142,7 +142,7 @@ public class TestFSSchedulerNode {
}
private void allocateContainers(FSSchedulerNode schedulerNode) {
- FairScheduler.assignPreemptedContainers(schedulerNode);
+ FairScheduler.attemptToAssignPreemptedResources(schedulerNode);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-8801. Fixed header comments
for docker utility functions. Contributed by Zian Chen
Posted by ha...@apache.org.
YARN-8801. Fixed header comments for docker utility functions.
Contributed by Zian Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa4bd493
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa4bd493
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa4bd493
Branch: refs/heads/YARN-1011
Commit: aa4bd493c309f09f8f2ea7449aa33c8b641fb8d2
Parents: 429a07e
Author: Eric Yang <ey...@apache.org>
Authored: Thu Sep 20 13:08:59 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Sep 20 13:08:59 2018 -0400
----------------------------------------------------------------------
.../container-executor/impl/utils/docker-util.h | 30 +++++++-------------
1 file changed, 10 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa4bd493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
index 278dc53..7b7322d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -81,8 +81,7 @@ char *get_docker_binary(const struct configuration *conf);
* Get the Docker command line string. The function will inspect the params file to determine the command to be run.
* @param command_file File containing the params for the Docker command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the Docker command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_command(const char* command_file, const struct configuration* conf, args *args);
@@ -98,8 +97,7 @@ int get_use_entry_point_flag();
* inspect command.
* @param command_file File containing the params for the Docker inspect command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the inspect command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_inspect_command(const char* command_file, const struct configuration* conf, args *args);
@@ -108,8 +106,7 @@ int get_docker_inspect_command(const char* command_file, const struct configurat
* Get the Docker load command line string. The function will verify that the params file is meant for the load command.
* @param command_file File containing the params for the Docker load command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the load command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_load_command(const char* command_file, const struct configuration* conf, args *args);
@@ -118,8 +115,7 @@ int get_docker_load_command(const char* command_file, const struct configuration
* Get the Docker pull command line string. The function will verify that the params file is meant for the pull command.
* @param command_file File containing the params for the Docker pull command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the pull command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_pull_command(const char* command_file, const struct configuration* conf, args *args);
@@ -128,8 +124,7 @@ int get_docker_pull_command(const char* command_file, const struct configuration
* Get the Docker rm command line string. The function will verify that the params file is meant for the rm command.
* @param command_file File containing the params for the Docker rm command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the rm command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_rm_command(const char* command_file, const struct configuration* conf, args *args);
@@ -138,8 +133,7 @@ int get_docker_rm_command(const char* command_file, const struct configuration*
* Get the Docker run command line string. The function will verify that the params file is meant for the run command.
* @param command_file File containing the params for the Docker run command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the run command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_run_command(const char* command_file, const struct configuration* conf, args *args);
@@ -148,8 +142,7 @@ int get_docker_run_command(const char* command_file, const struct configuration*
* Get the Docker stop command line string. The function will verify that the params file is meant for the stop command.
* @param command_file File containing the params for the Docker stop command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the stop command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_stop_command(const char* command_file, const struct configuration* conf, args *args);
@@ -158,8 +151,7 @@ int get_docker_stop_command(const char* command_file, const struct configuration
* Get the Docker kill command line string. The function will verify that the params file is meant for the kill command.
* @param command_file File containing the params for the Docker kill command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the kill command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_kill_command(const char* command_file, const struct configuration* conf, args *args);
@@ -169,8 +161,7 @@ int get_docker_kill_command(const char* command_file, const struct configuration
* params file is meant for the volume command.
* @param command_file File containing the params for the Docker volume command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the volume command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_volume_command(const char *command_file, const struct configuration *conf, args *args);
@@ -179,8 +170,7 @@ int get_docker_volume_command(const char *command_file, const struct configurati
* Get the Docker start command line string. The function will verify that the params file is meant for the start command.
* @param command_file File containing the params for the Docker start command
* @param conf Configuration struct containing the container-executor.cfg details
- * @param out Buffer to fill with the start command
- * @param outlen Size of the output buffer
+ * @param args Buffer to construct argv
* @return Return code with 0 indicating success and non-zero codes indicating error
*/
int get_docker_start_command(const char* command_file, const struct configuration* conf, args *args);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index a6b30b3..ad3413f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -23,17 +23,12 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.commons.lang3.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -43,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.log4j.Level;
@@ -84,13 +80,13 @@ public class TestContainerResourceUsage {
RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
- "Before app submittion, memory seconds should have been 0 but was "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() == 0);
+ "Before app submission, memory seconds should have been 0 but was "
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() == 0);
Assert.assertTrue(
"Before app submission, vcore seconds should have been 0 but was "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() == 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
@@ -105,7 +101,8 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
+ while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0
+ && cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
@@ -113,27 +110,35 @@ public class TestContainerResourceUsage {
rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
"While app is running, memory seconds should be >0 but is "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() > 0);
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() > 0);
Assert.assertTrue(
"While app is running, vcore seconds should be >0 but is "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() > 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() > 0);
MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ ru.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ ru.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ ru.getOpportunisticMemorySeconds(),
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ ru.getOpportunisticVcoreSeconds(),
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
}
- @Test (timeout = 120000)
+ @Test ()
public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
// Set max attempts to 1 so that when the first attempt fails, the app
// won't try to start a new one.
@@ -191,7 +196,7 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (app0.getRMAppMetrics().getMemorySeconds() <= 0
+ while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0
&& cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
@@ -216,19 +221,29 @@ public class TestContainerResourceUsage {
rm0.waitForState(nm, cId, RMContainerState.COMPLETED);
// Check that the container metrics match those from the app usage report.
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
for (RMContainer c : rmContainers) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics metricsBefore = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, metricsBefore.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, metricsBefore.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ metricsBefore.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ metricsBefore.getOpportunisticVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
MockRM rm1 = new MockRM(conf, memStore);
@@ -238,10 +253,22 @@ public class TestContainerResourceUsage {
// Compare container resource usage metrics from before and after restart.
RMAppMetrics metricsAfter = app0After.getRMAppMetrics();
- Assert.assertEquals("Vcore seconds were not the same after RM Restart",
- metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds());
- Assert.assertEquals("Memory seconds were not the same after RM Restart",
- metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds());
+ Assert.assertEquals(
+ "Guaranteed vcore seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedVcoreSeconds(),
+ metricsAfter.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(
+ "Guaranteed memory seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedMemorySeconds(),
+ metricsAfter.getGuaranteedMemorySeconds());
+ Assert.assertEquals(
+ "Opportunistic vcore seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticVcoreSeconds(),
+ metricsAfter.getOpportunisticVcoreSeconds());
+ Assert.assertEquals(
+ "Opportunistic memory seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticMemorySeconds(),
+ metricsAfter.getOpportunisticMemorySeconds());
rm0.stop();
rm0.close();
@@ -264,10 +291,8 @@ public class TestContainerResourceUsage {
MockRM rm = new MockRM(conf);
rm.start();
- RMApp app =
- rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
- null, "MAPREDUCE", false, keepRunningContainers);
+ RMApp app = rm.submitApp(200, "name", "user", new HashMap<>(), false,
+ "default", -1, null, "MAPREDUCE", false, keepRunningContainers);
MockNM nm =
new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
nm.registerNode();
@@ -275,18 +300,17 @@ public class TestContainerResourceUsage {
MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
int NUM_CONTAINERS = 1;
// allocate NUM_CONTAINERS containers
- am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
List<Container> containers =
- am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ am0.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
- containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers());
+ containers.addAll(am0.allocate(new ArrayList<>(),
+ new ArrayList<>()).getAllocatedContainers());
Thread.sleep(200);
}
@@ -294,26 +318,27 @@ public class TestContainerResourceUsage {
ContainerId containerId2 =
ContainerId.newContainerId(am0.getApplicationAttemptId(), 2);
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- containerId2.getContainerId(), ContainerState.RUNNING);
+ containerId2.getContainerId(), ContainerState.RUNNING);
rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
- Collection<RMContainer> rmContainers =
- rm.scheduler
- .getSchedulerAppInfo(am0.getApplicationAttemptId())
- .getLiveContainers();
+ Collection<RMContainer> rmContainers = rm.scheduler
+ .getSchedulerAppInfo(am0.getApplicationAttemptId())
+ .getLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId =
app.getCurrentAppAttempt().getMasterContainer().getId();
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- amContainerId.getContainerId(), ContainerState.COMPLETE);
+ amContainerId.getContainerId(), ContainerState.COMPLETE);
rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED);
rm.drainEvents();
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers) {
@@ -321,8 +346,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
if (c.getContainerId().equals(amContainerId)) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
} else {
// The remaining container should be RUNNING.
Assert.assertTrue("After first attempt failed, remaining container "
@@ -336,8 +363,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
}
@@ -355,14 +384,12 @@ public class TestContainerResourceUsage {
am1.registerAppAttempt();
rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// allocate NUM_CONTAINERS containers
- am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
- containers =
- am1.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ containers = am1.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
@@ -388,19 +415,26 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
- return;
}
private void waitforContainerCompletion(MockRM rm, MockNM nm,
@@ -419,16 +453,10 @@ public class TestContainerResourceUsage {
private AggregateAppResourceUsage calculateContainerResourceMetrics(
RMContainer rmContainer) {
- Resource resource = rmContainer.getContainer().getResource();
- long usedMillis =
- rmContainer.getFinishTime() - rmContainer.getCreationTime();
- long memorySeconds = resource.getMemorySize()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- long vcoreSeconds = resource.getVirtualCores()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- Map<String, Long> map = new HashMap<>();
- map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- return new AggregateAppResourceUsage(map);
+ ContainerResourceUsageReport resourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ return new AggregateAppResourceUsage(
+ resourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ resourceUsageReport.getOpportunisticResourceSecondsMap());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 342dab8..58be457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -209,7 +209,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
}
@Override
@@ -375,7 +375,7 @@ public abstract class MockAsm extends MockApps {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0,
- new HashMap<>());
+ new HashMap<>(), null, new HashMap<>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
index 5b0c34f..cdfd361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
@@ -479,7 +479,7 @@ public class TestCombinedSystemMetricsPublisher {
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(rmApp.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap,
- preemptedMap));
+ preemptedMap, new HashMap<>()));
when(rmApp.getApplicationTags()).thenReturn(
Collections.<String> emptySet());
ApplicationSubmissionContext appSubmissionContext =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index f2d82e9..a8f31e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -219,13 +219,25 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
- app.getRMAppMetrics().getMemorySeconds(),
+ app.getRMAppMetrics().getGuaranteedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getGuaranteedVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
- app.getRMAppMetrics().getVcoreSeconds(),
+ app.getRMAppMetrics().getOpportunisticMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getOpportunisticVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
app.getRMAppMetrics().getPreemptedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
@@ -517,7 +529,8 @@ public class TestSystemMetricsPublisher {
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics())
- .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
+ .thenReturn(new RMAppMetrics(null, 0, 0,
+ resourceMap, preemptedMap, new HashMap<>()));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 3b503e5..921d90c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -221,7 +221,7 @@ public class TestSystemMetricsPublisherForV2 {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(
- appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
+ appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0);
}
@Test(timeout = 10000)
@@ -365,14 +365,14 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus())
.thenReturn(FinalApplicationStatus.UNDEFINED);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
- resourceSecondsMap
+ guaranteedResourceSecondsMap
.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
- new HashMap<>()));
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
+ guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>()));
when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 3454d72..4f84567 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -202,7 +202,8 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -300,7 +301,8 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -377,7 +379,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -402,7 +405,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 11be3b1..12b6033 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -555,7 +555,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<>(),
+ new HashMap<>(0)));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -567,7 +568,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, new HashMap<>(), new HashMap<>());
+ finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -804,20 +805,33 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), 0L);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- 0L);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
+ Map<String, Long> guaranteedResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> opportunisticResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> preemptedResoureSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
- amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
+ amExitStatus, 0, guaranteedResourceSecondsMap,
+ preemptedResoureSecondsMap, opportunisticResourceSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 70887e0..6ee5e40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -243,7 +242,7 @@ public class TestRMAppTransitions {
ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
doReturn(null).when(resourceScheduler)
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
+ .getAppActiveResourceUsageReport(Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
@@ -1249,7 +1248,8 @@ public class TestRMAppTransitions {
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
- Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
+ Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT,
+ report.getApplicationResourceUsageReport());
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
Assert.assertTrue("bad proxy url for app",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..550e24d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -299,10 +299,12 @@ public class TestRMAppAttemptTransitions {
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L);
when(resourceScheduler
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
+ .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any()))
.thenReturn(appResUsgRpt);
spyRMContext = spy(rmContext);
Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
@@ -762,10 +764,12 @@ public class TestRMAppAttemptTransitions {
ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
- when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
- .thenReturn(appResUsgRpt);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L);
+ when(scheduler.getAppActiveResourceUsageReport(
+ any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
// start and finish the attempt
Container amContainer = allocateApplicationAttempt();
@@ -777,12 +781,14 @@ public class TestRMAppAttemptTransitions {
// expect usage stats to come from the scheduler report
ApplicationResourceUsageReport report =
applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(123456L, report.getMemorySeconds());
- Assert.assertEquals(55544L, report.getVcoreSeconds());
+ Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds());
+ Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds());
// finish app attempt and remove it from scheduler
- when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L);
sendAttemptUpdateSavedEvent(applicationAttempt);
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
@@ -793,8 +799,8 @@ public class TestRMAppAttemptTransitions {
when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
report = applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(223456, report.getMemorySeconds());
- Assert.assertEquals(75544, report.getVcoreSeconds());
+ Assert.assertEquals(223456, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index def0419..943a59a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -2838,7 +2838,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
Thread.sleep(1000);
AllocateResponse allocResponse = am1.schedule();
ApplicationResourceUsageReport report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
@@ -2851,7 +2851,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
allocResponse = am1.schedule();
}
report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index e239191..7a10d19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -223,23 +223,23 @@ public class TestCapacitySchedulerNodeLabelUpdate {
RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
ApplicationResourceUsageReport appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(1024, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(1, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
// request a container.
am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED);
appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(2048, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(2, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
LeafQueue queue =
(LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
.getQueue("a");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index f88ac8b..de97ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -66,7 +66,7 @@ public class TestAppPage {
RMAppMetrics appMetrics =
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 93377be..3f5926a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Token;
@@ -259,7 +258,7 @@ public class TestRMWebApp {
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getLaunchTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
- (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
+ null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index cea0088..87846c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -138,7 +138,7 @@ public class TestRMWebAppFairScheduler {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
- new HashMap<>(), new HashMap<>());
+ new HashMap<>(), new HashMap<>(), new HashMap<>());
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 15f94e1..8eb6023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1605,7 +1605,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
throws JSONException, Exception {
- int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0);
+ int expectedNumberOfElements = 45 + (hasResourceReqs ? 2 : 0);
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 40bdbd8..6f1ee3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -339,12 +339,22 @@ public final class RouterWebServiceUtil {
am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB());
am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores());
+ am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() +
+ uam.getAllocatedOpportunisticMB());
+ am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() +
+ uam.getAllocatedOpportunisticVCores());
am.setReservedMB(am.getReservedMB() + uam.getReservedMB());
am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB());
am.setRunningContainers(
am.getRunningContainers() + uam.getRunningContainers());
- am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds());
- am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds());
+ am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() +
+ uam.getGuaranteedMemorySeconds());
+ am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() +
+ uam.getGuaranteedVcoreSeconds());
+ am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() +
+ uam.getOpportunisticVcoreSeconds());
+ am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() +
+ uam.getOpportunisticMemorySeconds());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index edf3804..1a5c42e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -249,8 +249,10 @@ public class TestRouterWebServiceUtil {
Assert.assertEquals(value * 3, app.getReservedMB());
Assert.assertEquals(value * 3, app.getReservedVCores());
Assert.assertEquals(value * 3, app.getRunningContainers());
- Assert.assertEquals(value * 3, app.getMemorySeconds());
- Assert.assertEquals(value * 3, app.getVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds());
Assert.assertEquals(3, app.getResourceRequests().size());
}
@@ -258,11 +260,15 @@ public class TestRouterWebServiceUtil {
am.getResourceRequests().add(new ResourceRequestInfo());
am.setAllocatedMB(value);
am.setAllocatedVCores(value);
+ am.setAllocatedOpportunisticVCores(value);
+ am.setAllocatedOpportunisticMB(value);
am.setReservedMB(value);
am.setReservedVCores(value);
am.setRunningContainers(value);
- am.setMemorySeconds(value);
- am.setVcoreSeconds(value);
+ am.setGuaranteedMemorySeconds(value);
+ am.setGuaranteedVcoreSeconds(value);
+ am.setOpportunisticMemorySeconds(value);
+ am.setOpportunisticVcoreSeconds(value);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-6672. Add NM preemption of
opportunistic containers when utilization goes high.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
index 183d868..a2d4aa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -20,12 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -36,11 +33,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
@@ -53,8 +47,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
@@ -78,8 +70,6 @@ import java.util.concurrent.ConcurrentHashMap;
public class TestContainerSchedulerWithOverAllocation
extends BaseContainerManagerTest {
private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
- private static final int NM_CONTAINERS_VCORES = 4;
- private static final int NM_CONTAINERS_MEMORY_MB = 2048;
static {
LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
@@ -115,6 +105,11 @@ public class TestContainerSchedulerWithOverAllocation
conf.setFloat(
YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
0.75f);
+ conf.setFloat(YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
+ 0.8f);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD, 0.8f);
+ conf.setInt(YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT, 2);
// disable the monitor thread in ContainersMonitor to allow control over
// when opportunistic containers are launched with over-allocation
conf.setBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false);
@@ -134,9 +129,9 @@ public class TestContainerSchedulerWithOverAllocation
StartContainersRequest allRequests = StartContainersRequest.newInstance(
new ArrayList<StartContainerRequest>() { {
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
} }
);
containerManager.startContainers(allRequests);
@@ -171,9 +166,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
+ BuilderUtils.newResource(824, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -191,11 +186,9 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC))
));
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// this container is not expected to be started immediately because
// opportunistic containers cannot be started if the node would be
@@ -212,8 +205,8 @@ public class TestContainerSchedulerWithOverAllocation
});
// try to start opportunistic containers out of band.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// this container is expected to be started immediately because there
// are (memory: 1024, vcore: 0.625) available based on over-allocation
@@ -247,9 +240,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
+ BuilderUtils.newResource(824, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -267,17 +260,14 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC))
));
// try to start opportunistic containers out of band because they can
// not be launched at container scheduler event if the node would be
// over-allocated.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// this container will not start immediately because there is not
// enough resource available at the moment either in terms of
@@ -312,9 +302,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -331,11 +321,9 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC))
));
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// this container is not expected to be started immediately because
// opportunistic containers cannot be started if the node would be
@@ -352,8 +340,8 @@ public class TestContainerSchedulerWithOverAllocation
});
// try to start opportunistic containers out of band.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// this container is expected to be started because there is resources
// available because the actual utilization is very low
@@ -384,9 +372,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -406,13 +394,11 @@ public class TestContainerSchedulerWithOverAllocation
for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
moreContainerRequests.add(
createStartContainerRequest(2 + a,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
}
containerManager.startContainers(
StartContainersRequest.newInstance(moreContainerRequests));
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// All OPPORTUNISTIC containers but the last one should be queued.
// The last OPPORTUNISTIC container to launch should be killed.
@@ -454,9 +440,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1200, 1), true));
+ BuilderUtils.newResource(1200, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(400, 1), true));
+ BuilderUtils.newResource(400, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -473,7 +459,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(400, 1), false))
+ BuilderUtils.newResource(400, 1), ExecutionType.OPPORTUNISTIC))
));
// the OPPORTUNISTIC container can be safely launched even though
@@ -507,9 +493,11 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
+ BuilderUtils.newResource(824, 1),
+ ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -527,7 +515,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
+ BuilderUtils.newResource(512, 1), ExecutionType.GUARANTEED))
));
// the GUARANTEED container is expected be launched immediately without
@@ -561,9 +549,11 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
+ BuilderUtils.newResource(824, 1),
+ ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -581,7 +571,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
+ BuilderUtils.newResource(512, 1), ExecutionType.GUARANTEED))
));
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -631,16 +621,18 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
}
}
));
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// Two OPPORTUNISTIC containers are expected to start with the
// unallocated resources, but one will be queued because no
@@ -655,8 +647,8 @@ public class TestContainerSchedulerWithOverAllocation
// try to start the opportunistic container out of band because it can
// not be launched at container scheduler event if the node would be
// over-allocated.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// now the queued opportunistic container should also start
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -671,7 +663,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), true))
+ BuilderUtils.newResource(512, 1), ExecutionType.GUARANTEED))
));
// the GUARANTEED container is expected be launched immediately without
@@ -690,83 +682,6 @@ public class TestContainerSchedulerWithOverAllocation
}
/**
- * Start four OPPORTUNISTIC containers which in aggregates exceeds the
- * capacity of the node. The real resource utilization of the first two
- * OPPORTUNISTIC containers are high whereas that of the latter two are
- * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
- * container will eventually start running after preempting the third
- * and fourth OPPORTUNISTIC container (which releases no resources) and
- * then the second OPPORTUNISTIC container.
- */
- public void
- testKillOppContainersConservativelyWithOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(3,
- BuilderUtils.newResource(1024, 1), false));
- }
- }
- ));
- // All four GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
-
- // the containers utilization is at the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- // try to start a GUARANTEED container when there's nothing left unallocated
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(4,
- BuilderUtils.newResource(1024, 1), true))
- ));
-
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
- GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
- newInstance(new ArrayList<ContainerId>() {
- {
- add(createContainerId(0));
- add(createContainerId(1));
- add(createContainerId(2));
- add(createContainerId(3));
- add(createContainerId(4));
- }
- });
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
- for (ContainerStatus status : containerStatuses) {
- if (status.getContainerId().equals(createContainerId(0)) ||
- status.getContainerId().equals(createContainerId(4))) {
- Assert.assertEquals(
- ContainerSubState.RUNNING, status.getContainerSubState());
- } else {
- Assert.assertTrue(status.getDiagnostics().contains(
- "Container Killed to make room for Guaranteed Container"));
- }
- System.out.println("\nStatus : [" + status + "]\n");
- }
- }
-
- /**
* Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
* which in aggregate exceeds the capacity of the node. The first two
* OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
@@ -784,11 +699,11 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -810,9 +725,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(4,
- BuilderUtils.newResource(800, 1), false));
+ BuilderUtils.newResource(800, 1), ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -831,8 +746,7 @@ public class TestContainerSchedulerWithOverAllocation
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.DONE);
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// only one OPPORTUNISTIC container is start because no over-allocation
// is allowed to start OPPORTUNISTIC containers at container finish event.
@@ -854,10 +768,8 @@ public class TestContainerSchedulerWithOverAllocation
// now try to start the OPPORTUNISTIC container that was queued because
// we don't start OPPORTUNISTIC containers at container finish event if
// the node would be over-allocated
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(4), ContainerSubState.RUNNING);
verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
@@ -884,7 +796,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(0,
- BuilderUtils.newResource(2048, 4), true))));
+ BuilderUtils.newResource(2048, 4), ExecutionType.GUARANTEED))));
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(0), ContainerSubState.RUNNING);
@@ -897,9 +809,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -913,8 +825,8 @@ public class TestContainerSchedulerWithOverAllocation
ResourceUtilization.newInstance(1536, 0, 1.0f/2));
// try to start opportunistic containers out of band.
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// no containers in queue are expected to be launched because the
// containers utilization is not below the over-allocation threshold
@@ -934,8 +846,8 @@ public class TestContainerSchedulerWithOverAllocation
setContainerResourceUtilization(
ResourceUtilization.newInstance(512, 0, 1.0f/8));
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// the two OPPORTUNISTIC containers are expected to be launched
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -952,11 +864,332 @@ public class TestContainerSchedulerWithOverAllocation
});
}
+ /**
+ * Start a GUARANTEED container, an OPPORTUNISTIC container, a GUARANTEED
+ * container and another OPPORTUNISTIC container in order. When the node
+ * memory utilization is over its preemption threshold, the two OPPORTUNISTIC
+ * containers should be killed.
+ */
+ @Test
+ public void testPreemptOpportunisticContainersUponHighMemoryUtilization()
+ throws Exception {
+ containerManager.start();
+
+ // try to start four containers at once. the first GUARANTEED container
+ // that requests (1024 MB, 1 vcore) can be launched because there is
+ // (2048 MB, 4 vcores) unallocated. The second container, which is
+ // OPPORTUNISTIC, can also be launched because it asks for 512 MB, 1 vcore
+ // which is less than what is left unallocated after launching the first
+ // one GUARANTEED container, (1024 MB, 3 vcores).
+ // The 3rd container, which is GUARANTEED, can also be launched because
+ // the node resource utilization utilization is zero such that
+ // over-allocation kicks in. The 4th one, an OPPORTUNISTIC container,
+ // will be queued because OPPORTUNISTIC containers can only be
+ // launched when node resource utilization is checked, if launching them
+ // would cause node over-allocation.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(300, 1), ExecutionType.OPPORTUNISTIC));
+ }
+ }
+ ));
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
+
+ // the first three containers are all expected to start
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.SCHEDULED);
+
+ // try to check node resource utilization and start the second
+ // opportunistic containers out of band. Because the node resource
+ // utilization is zero at the moment, over-allocation will kick in
+ // and the container will be launched.
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ // the containers memory utilization is over the preemption threshold
+ // (2048 > 2048 * 0.8 = 1638.4)
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 0.5f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // (2048 - 2048 * 0.8) = 409.6 MB of memory needs to be reclaimed,
+ // which shall result in both OPPORTUNISTIC containers to be preempted.
+ // (Preempting the most recently launched OPPORTUNISTIC container, that
+ // is the 4th container, would only release at most 300 MB of memory)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.DONE);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.DONE);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ }
+
+ /**
+ * Start a GUARANTEED container followed by an OPPORTUNISTIC container, which
+ * in aggregates does not take more than the capacity of the node.
+ * When the node memory utilization is above the preemption threshold, the
+ * OPPORTUNISTIC container should not be killed because the node is not being
+ * over-allocated.
+ */
+ @Test
+ public void testNoPreemptionUponHighMemoryUtilizationButNoOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ // start two containers, one GUARANTEED and one OPPORTUNISTIC, that together
+ // take up all the allocations (2048 MB of memory and 4 vcores available on
+ // the node). They can be both launched immediately because there are enough
+ // allocations to do so. When the two containers fully utilize their
+ // resource requests, that is, the node is being 100% utilized, the
+ // OPPORTUNISTIC container shall continue to run because the node is
+ // not be over-allocated.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 2),
+ ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 2),
+ ExecutionType.OPPORTUNISTIC));
+ }
+ }
+ ));
+ // both containers shall be launched immediately because there are
+ // enough allocations to do so
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the node is being fully utilized, which is above the preemption
+ // threshold (2048 * 0.75 = 1536 MB, 1.0f)
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // no containers shall be preempted because the node is not being
+ // over-allocated so it is safe to allow the node to be fully utilized
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start a GUARANTEED container, an OPPORTUNISTIC container, a GUARANTEED
+ * container and another OPPORTUNISTIC container in order. When the node
+ * cpu utilization is over its preemption threshold a few times in a row,
+ * the two OPPORTUNISTIC containers should be killed one by one.
+ */
+ @Test
+ public void testPreemptionUponHighCPUUtilization() throws Exception {
+ containerManager.start();
+
+ // try to start 4 containers at once. The first container, can be
+ // safely launched immediately (2048 MB, 4 vcores left unallocated).
+ // The second container, can also be launched immediately, because
+ // there is enough resources unallocated after launching the first
+ // container (2048 - 512 = 1536 MB, 4 - 2 = 2 vcores). After launching
+ // the first two containers, there are 1024 MBs of memory and 1 vcore
+ // left unallocated, so there is not enough allocation to launch the
+ // third container. But because the third container is GUARANTEED and
+ // the node resource utilization is zero, we can launch it based on
+ // over-allocation (the projected resource utilization will be 512 MB
+ // of memory and 2 vcores, below the over-allocation threshold)
+ // The fourth container, which is OPPORTUNISTIC, will be queued because
+ // OPPORTUNISTIC containers can not be launched based on over-allocation
+ // upon container start requests (they can only be launched when node
+ // resource utilization is checked in ContainersMonitor)
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(512, 2), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 2), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
+ }
+ }
+ ));
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
+ // the first three containers are expected to start. The first two
+ // can be launched based on free allocation, the third can be
+ // launched based on over-allocation
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // try to start second opportunistic containers out of band.
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // the second opportunistic container is expected to start because
+ // the node resource utilization is at zero, the projected utilization
+ // is 512 MBs of memory and 1 vcore
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ final float fullCpuUtilization = 1.0f;
+
+ // the containers CPU utilization is over its preemption threshold (0.8f)
+ // for the first time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // all containers should continue to be running because we don't
+ // preempt OPPORTUNISTIC containers right away
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+
+ // the containers CPU utilization is over its preemption threshold (0.8f)
+ // for the second time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // all containers should continue to be running because we don't preempt
+ // OPPORTUNISTIC containers when the cpu is over the preemption threshold
+ // (0.8f) the second time
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+
+ // the containers CPU utilization is over the preemption threshold (0.8f)
+ // for the third time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // because CPU utilization is over its preemption threshold three times
+ // consecutively, the amount of cpu utilization over the preemption
+ // threshold, that is, 1.0 - 0.8 = 0.2f CPU needs to be reclaimed and
+ // as a result, the most recently launched OPPORTUNISTIC container should
+ // be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.DONE);
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ // again, the containers CPU utilization is over the preemption threshold
+ // (0.8f) for the first time (the cpu over-limit count is reset every time
+ // a preemption is triggered)
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // no CPU resource is expected to be reclaimed when the CPU utilization
+ // goes over the preemption threshold the first time
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ // the containers CPU utilization is over the preemption threshold (0.9f)
+ // for the second time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // still no CPU resource is expected to be reclaimed when the CPU
+ // utilization goes over the preemption threshold the second time
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ // the containers CPU utilization is over the preemption threshold
+ // for the third time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // because CPU utilization is over its preemption threshold three times
+ // consecutively, the amount of cpu utilization over the preemption
+ // threshold, that is, 1.0 - 0.8 = 0.2f CPU needs to be reclaimed and
+ // as a result, the other OPPORTUNISTIC container should be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.DONE);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+ }
+
private void setContainerResourceUtilization(ResourceUtilization usage) {
- ((ContainerMonitorForOverallocationTest)
- containerManager.getContainersMonitor())
- .setContainerResourceUsage(usage);
+ ((ContainerMonitorForTest) containerManager.getContainersMonitor())
+ .setContainerResourceUsage(usage);
}
private void allowContainerToSucceed(int containerId) {
@@ -965,12 +1198,11 @@ public class TestContainerSchedulerWithOverAllocation
}
- protected StartContainerRequest createStartContainerRequest(int containerId,
- Resource resource, boolean isGuaranteed) throws IOException {
+ protected StartContainerRequest createStartContainerRequest(
+ int containerId, Resource resource, ExecutionType executionType)
+ throws IOException {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
- ExecutionType.OPPORTUNISTIC;
Token containerToken = createContainerToken(
createContainerId(containerId),
DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
@@ -1004,9 +1236,7 @@ public class TestContainerSchedulerWithOverAllocation
* container processes for testing purposes.
*/
private static class LongRunningContainerSimulatingContainersManager
- extends ContainerManagerImpl {
-
- private final String user;
+ extends ContainerManagerForTest {
LongRunningContainerSimulatingContainersManager(
Context context, ContainerExecutor exec,
@@ -1014,27 +1244,8 @@ public class TestContainerSchedulerWithOverAllocation
NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics,
LocalDirsHandlerService dirsHandler, String user) {
- super(context, exec, deletionContext,
- nodeStatusUpdater, metrics, dirsHandler);
- this.user = user;
- }
-
- @Override
- protected UserGroupInformation getRemoteUgi() throws YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
- .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
- .getKeyId()));
- return ugi;
- }
-
- @Override
- protected AsyncDispatcher createDispatcher() {
- return new DrainDispatcher();
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler, user);
}
/**
@@ -1060,27 +1271,10 @@ public class TestContainerSchedulerWithOverAllocation
throws Exception {
return "123";
}
-
};
}
};
}
-
- @Override
- protected ContainersMonitor createContainersMonitor(
- ContainerExecutor exec) {
- return new ContainerMonitorForOverallocationTest(exec,
- dispatcher, context);
- }
-
- public void startContainersOutOfBandUponLowUtilization() {
- ((ContainerMonitorForOverallocationTest) getContainersMonitor())
- .attemptToStartContainersUponLowUtilization();
- }
-
- public void drainAsyncEvents() {
- ((DrainDispatcher) dispatcher).await();
- }
}
/**
@@ -1176,56 +1370,4 @@ public class TestContainerSchedulerWithOverAllocation
}
}
}
-
- /**
- * A test implementation of container monitor that allows control of
- * current resource utilization.
- */
- private static class ContainerMonitorForOverallocationTest
- extends ContainersMonitorImpl {
-
- private ResourceUtilization containerResourceUsage =
- ResourceUtilization.newInstance(0, 0, 0.0f);
-
- ContainerMonitorForOverallocationTest(ContainerExecutor exec,
- AsyncDispatcher dispatcher, Context context) {
- super(exec, dispatcher, context);
- }
-
- @Override
- public long getPmemAllocatedForContainers() {
- return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
- }
-
- @Override
- public long getVmemAllocatedForContainers() {
- float pmemRatio = getConfig().getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- return (long) (pmemRatio * getPmemAllocatedForContainers());
- }
-
- @Override
- public long getVCoresAllocatedForContainers() {
- return NM_CONTAINERS_VCORES;
- }
-
- @Override
- public ContainersResourceUtilization getContainersUtilization(
- boolean latest) {
- return new ContainersMonitor.ContainersResourceUtilization(
- containerResourceUsage, System.currentTimeMillis());
- }
-
- @Override
- protected void checkOverAllocationPrerequisites() {
- // do not check
- }
-
-
- public void setContainerResourceUsage(
- ResourceUtilization containerResourceUsage) {
- this.containerResourceUsage = containerResourceUsage;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java
new file mode 100644
index 0000000..bbc7c49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link SnapshotBasedOverAllocationPreemptionPolicy}.
+ */
+public class TestSnapshotBasedOverAllocationPreemptionPolicy {
+ // Both the CPU preemption threshold and the memory preemption threshold
+ // are 75%
+ private final static ResourceThresholds PREEMPTION_THRESHOLDS =
+ ResourceThresholds.newInstance(0.75f, 0.75f);
+
+ // The CPU utilization is allowed to go over the cpu preemption threshold
+ // 2 times in a row before any container is preempted to reclaim cpu resources
+ private final static int MAX_CPU_OVER_PREEMPTION_THRESHOLDS = 2;
+
+ private final ContainersMonitor containersMonitor =
+ mock(ContainersMonitor.class);
+
+ @Before
+ public void setUp() {
+ // the node has an allocation of 2048 MB of memory
+ when(containersMonitor.getPmemAllocatedForContainers()).
+ thenReturn(2048 * 1024 * 1024L);
+ }
+
+ /**
+ * The memory utilization goes above its preemption threshold,
+ * 2048 * 0.75f = 1536 MB (the node has an allocation of 2048 MB memory).
+ */
+ @Test
+ public void testMemoryOverPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current memory utilization, 2000 MB is over the preemption
+ // threshold, 2048 * 0.75, which is 1536 MB. The CPU utilization,
+ // 0.5f is below the preemption threshold, 0.75f.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(2000, 0, 0.5f),
+ Time.now()));
+
+ // the amount of memory utilization over the preemption threshold, that is,
+ // 2000 - (2048 * 0.75) = 464 MB of memory, shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(464, 0, 0f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+
+ /**
+ * The CPU utilization goes above its preemption threshold, 0.75f.
+ */
+ @Test
+ public void testCpuOverPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f, for the first time. The memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 0.5f, is below the preemption threshold,
+ // 0.75f. In the meantime the memory utilization, 1000 MB is also below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 0.5f),
+ Time.now()));
+ // no resources shall be reclaimed
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f. In the meantime the memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the first time
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is again over the preemption
+ // threshold, 0.75f. In the meantime the memory utilization, 1000 MB
+ // is below the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the second time in a row
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // the third time in a row. In the meantime the memory utilization, 1000 MB
+ // is below the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // the amount of cpu utilization over the preemption threshold, that is,
+ // 1.0 - 0.75f = 0.25, shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.25f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+
+ /**
+ * Both memory and CPU utilization go over their preemption thresholds
+ * respectively.
+ */
+ @Test
+ public void testMemoryCpuOverPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f, for the first time. The memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the first time.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 0.5f, is below the preemption threshold,
+ // 0.75f. The memory utilization, 2000 MB, however, is above the memory
+ // preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(2000, 0, 0.5f),
+ Time.now()));
+ // the amount of memory utilization over the preemption threshold, that is,
+ // 2000 - (2048 * 0.75) = 464 MB of memory, shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(464, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f, for the first time. The memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the first time.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is again over the preemption
+ // threshold, 0.75f. In the meantime the memory utilization, 1000 MB
+ // is still below the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the second time in a row.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the CPU preemption threshold,
+ // 0.75f, the third time in a row. In the meantime, the memory utilization,
+ // 2000 MB, is also over the memory preemption threshold,
+ // 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(2000, 0, 1.0f),
+ Time.now()));
+ // the amount of memory utilization over the preemption threshold, that is,
+ // 2000 - (2048 * 0.75) = 464 MB of memory, and the amount of cpu
+ // utilization over the preemption threshold, that is, 1.0f - 0.75f = 0.25f,
+ // shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(464, 0, 0.25f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+
+ /**
+ * Both memory and CPU utilization are under their preemption thresholds.
+ */
+ @Test
+ public void testBothMemoryAndCpuUnderPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current CPU utilization, 0.5f, is below the preemption threshold,
+ // 0.75f. In the meantime the memory utilization, 1000 MB is also below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 0.5f),
+ Time.now()));
+ // no resources shall be reclaimed because both CPU and memory utilization
+ // are under the preemption threshold
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDDS-394. Rename *Key Apis in
DatanodeContainerProtocol to *Block apis. Contributed Dinesh Chitlangia.
Posted by ha...@apache.org.
HDDS-394. Rename *Key Apis in DatanodeContainerProtocol to *Block apis.
Contributed Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/096a7160
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/096a7160
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/096a7160
Branch: refs/heads/YARN-1011
Commit: 096a7160803494219581c067dfcdb67d2bd0bcdb
Parents: aa4bd49
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Sep 20 11:51:49 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Sep 20 11:51:49 2018 -0700
----------------------------------------------------------------------
.../hdds/scm/storage/ChunkOutputStream.java | 13 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 8 +-
.../scm/storage/ContainerProtocolCalls.java | 62 ++---
.../container/common/helpers/BlockData.java | 255 +++++++++++++++++++
.../ozone/container/common/helpers/KeyData.java | 253 ------------------
.../main/proto/DatanodeContainerProtocol.proto | 74 +++---
.../common/impl/OpenContainerBlockMap.java | 46 ++--
.../DeleteBlocksCommandHandler.java | 4 +-
.../server/ratis/ContainerStateMachine.java | 28 +-
.../keyvalue/KeyValueBlockIterator.java | 16 +-
.../container/keyvalue/KeyValueContainer.java | 4 +-
.../container/keyvalue/KeyValueHandler.java | 124 ++++-----
.../container/keyvalue/helpers/BlockUtils.java | 199 +++++++++++++++
.../container/keyvalue/helpers/KeyUtils.java | 199 ---------------
.../keyvalue/helpers/KeyValueContainerUtil.java | 12 +-
.../keyvalue/helpers/SmallFileUtils.java | 2 +-
.../keyvalue/impl/BlockManagerImpl.java | 229 +++++++++++++++++
.../container/keyvalue/impl/KeyManagerImpl.java | 227 -----------------
.../container/keyvalue/impl/package-info.java | 5 +-
.../keyvalue/interfaces/BlockManager.java | 84 ++++++
.../keyvalue/interfaces/KeyManager.java | 84 ------
.../keyvalue/interfaces/package-info.java | 21 ++
.../background/BlockDeletingService.java | 10 +-
.../keyvalue/TestBlockManagerImpl.java | 211 +++++++++++++++
.../keyvalue/TestChunkManagerImpl.java | 2 +-
.../container/keyvalue/TestKeyManagerImpl.java | 191 --------------
.../keyvalue/TestKeyValueBlockIterator.java | 30 +--
.../keyvalue/TestKeyValueContainer.java | 26 +-
.../container/keyvalue/TestKeyValueHandler.java | 38 +--
.../ozone/client/io/ChunkGroupInputStream.java | 6 +-
.../TestStorageContainerManagerHelper.java | 8 +-
.../ozone/client/rest/TestOzoneRestClient.java | 8 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 8 +-
.../ozone/container/ContainerTestHelper.java | 84 +++---
.../container/TestContainerReplication.java | 24 +-
.../common/TestBlockDeletingService.java | 12 +-
.../container/common/helpers/TestBlockData.java | 127 +++++++++
.../container/common/helpers/TestKeyData.java | 119 ---------
.../common/impl/TestCloseContainerHandler.java | 51 ++--
.../common/impl/TestContainerPersistence.java | 154 +++++------
.../commandhandler/TestBlockDeletion.java | 9 +-
.../container/ozoneimpl/TestOzoneContainer.java | 100 ++++----
.../server/TestContainerStateMachine.java | 2 +-
.../hadoop/ozone/om/TestOzoneManager.java | 4 +-
.../ozone/scm/TestContainerSmallFile.java | 4 +-
.../TestGetCommittedBlockLengthAndPutKey.java | 12 +-
.../hadoop/ozone/web/client/TestKeys.java | 44 ++--
.../hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../ozone/om/ScmBlockLocationTestIngClient.java | 2 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 42 +--
50 files changed, 1680 insertions(+), 1599 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
index 8d311d0..10b3bb5 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkOutputStream.java
@@ -23,7 +23,7 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyData;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.client.BlockID;
@@ -32,7 +32,8 @@ import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.UUID;
-import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.putKey;
+import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
+ .putBlock;
import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
.writeChunk;
@@ -57,7 +58,7 @@ public class ChunkOutputStream extends OutputStream {
private final BlockID blockID;
private final String key;
private final String traceID;
- private final KeyData.Builder containerKeyData;
+ private final BlockData.Builder containerBlockData;
private XceiverClientManager xceiverClientManager;
private XceiverClientSpi xceiverClient;
private ByteBuffer buffer;
@@ -84,7 +85,7 @@ public class ChunkOutputStream extends OutputStream {
this.chunkSize = chunkSize;
KeyValue keyValue = KeyValue.newBuilder()
.setKey("TYPE").setValue("KEY").build();
- this.containerKeyData = KeyData.newBuilder()
+ this.containerBlockData = BlockData.newBuilder()
.setBlockID(blockID.getDatanodeBlockIDProtobuf())
.addMetadata(keyValue);
this.xceiverClientManager = xceiverClientManager;
@@ -154,7 +155,7 @@ public class ChunkOutputStream extends OutputStream {
writeChunkToContainer();
}
try {
- putKey(xceiverClient, containerKeyData.build(), traceID);
+ putBlock(xceiverClient, containerBlockData.build(), traceID);
} catch (IOException e) {
throw new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
@@ -230,6 +231,6 @@ public class ChunkOutputStream extends OutputStream {
throw new IOException(
"Unexpected Storage Container Exception: " + e.toString(), e);
}
- containerKeyData.addChunks(chunk);
+ containerBlockData.addChunks(chunk);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index 33bf90c..db9d374 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -326,8 +326,8 @@ public final class HddsUtils {
switch (proto.getCmdType()) {
case ReadContainer:
case ReadChunk:
- case ListKey:
- case GetKey:
+ case ListBlock:
+ case GetBlock:
case GetSmallFile:
case ListContainer:
case ListChunk:
@@ -340,8 +340,8 @@ public final class HddsUtils {
case CreateContainer:
case DeleteChunk:
case DeleteContainer:
- case DeleteKey:
- case PutKey:
+ case DeleteBlock:
+ case PutBlock:
case PutSmallFile:
default:
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index 1d6a89d..6b7a328 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -35,16 +35,16 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.DatanodeBlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .GetKeyRequestProto;
+ .GetBlockRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .GetKeyResponseProto;
+ .GetBlockResponseProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.GetSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.GetSmallFileResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyData;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .PutKeyRequestProto;
+ .PutBlockRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.PutSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -76,33 +76,33 @@ public final class ContainerProtocolCalls {
}
/**
- * Calls the container protocol to get a container key.
+ * Calls the container protocol to get a container block.
*
* @param xceiverClient client to perform call
* @param datanodeBlockID blockID to identify container
* @param traceID container protocol call args
- * @return container protocol get key response
+ * @return container protocol get block response
* @throws IOException if there is an I/O error while performing the call
*/
- public static GetKeyResponseProto getKey(XceiverClientSpi xceiverClient,
+ public static GetBlockResponseProto getBlock(XceiverClientSpi xceiverClient,
DatanodeBlockID datanodeBlockID, String traceID) throws IOException {
- GetKeyRequestProto.Builder readKeyRequest = GetKeyRequestProto
+ GetBlockRequestProto.Builder readBlockRequest = GetBlockRequestProto
.newBuilder()
.setBlockID(datanodeBlockID);
String id = xceiverClient.getPipeline().getLeader().getUuidString();
ContainerCommandRequestProto request = ContainerCommandRequestProto
.newBuilder()
- .setCmdType(Type.GetKey)
+ .setCmdType(Type.GetBlock)
.setContainerID(datanodeBlockID.getContainerID())
.setTraceID(traceID)
.setDatanodeUuid(id)
- .setGetKey(readKeyRequest)
+ .setGetBlock(readBlockRequest)
.build();
ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
validateContainerResponse(response);
- return response.getGetKey();
+ return response.getGetBlock();
}
/**
@@ -136,26 +136,26 @@ public final class ContainerProtocolCalls {
}
/**
- * Calls the container protocol to put a container key.
+ * Calls the container protocol to put a container block.
*
* @param xceiverClient client to perform call
- * @param containerKeyData key data to identify container
+ * @param containerBlockData block data to identify container
* @param traceID container protocol call args
* @throws IOException if there is an I/O error while performing the call
*/
- public static void putKey(XceiverClientSpi xceiverClient,
- KeyData containerKeyData, String traceID) throws IOException {
- PutKeyRequestProto.Builder createKeyRequest = PutKeyRequestProto
+ public static void putBlock(XceiverClientSpi xceiverClient,
+ BlockData containerBlockData, String traceID) throws IOException {
+ PutBlockRequestProto.Builder createBlockRequest = PutBlockRequestProto
.newBuilder()
- .setKeyData(containerKeyData);
+ .setBlockData(containerBlockData);
String id = xceiverClient.getPipeline().getLeader().getUuidString();
ContainerCommandRequestProto request = ContainerCommandRequestProto
.newBuilder()
- .setCmdType(Type.PutKey)
- .setContainerID(containerKeyData.getBlockID().getContainerID())
+ .setCmdType(Type.PutBlock)
+ .setContainerID(containerBlockData.getBlockID().getContainerID())
.setTraceID(traceID)
.setDatanodeUuid(id)
- .setPutKey(createKeyRequest)
+ .setPutBlock(createBlockRequest)
.build();
ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
validateContainerResponse(response);
@@ -224,9 +224,9 @@ public final class ContainerProtocolCalls {
/**
* Allows writing a small file using single RPC. This takes the container
- * name, key name and data to write sends all that data to the container using
- * a single RPC. This API is designed to be used for files which are smaller
- * than 1 MB.
+ * name, block name and data to write sends all that data to the container
+ * using a single RPC. This API is designed to be used for files which are
+ * smaller than 1 MB.
*
* @param client - client that communicates with the container.
* @param blockID - ID of the block
@@ -238,12 +238,12 @@ public final class ContainerProtocolCalls {
BlockID blockID, byte[] data, String traceID)
throws IOException {
- KeyData containerKeyData =
- KeyData.newBuilder().setBlockID(blockID.getDatanodeBlockIDProtobuf())
+ BlockData containerBlockData =
+ BlockData.newBuilder().setBlockID(blockID.getDatanodeBlockIDProtobuf())
.build();
- PutKeyRequestProto.Builder createKeyRequest =
- PutKeyRequestProto.newBuilder()
- .setKeyData(containerKeyData);
+ PutBlockRequestProto.Builder createBlockRequest =
+ PutBlockRequestProto.newBuilder()
+ .setBlockData(containerBlockData);
KeyValue keyValue =
KeyValue.newBuilder().setKey("OverWriteRequested").setValue("true")
@@ -255,7 +255,7 @@ public final class ContainerProtocolCalls {
PutSmallFileRequestProto putSmallFileRequest =
PutSmallFileRequestProto.newBuilder().setChunkInfo(chunk)
- .setKey(createKeyRequest).setData(ByteString.copyFrom(data))
+ .setBlock(createBlockRequest).setData(ByteString.copyFrom(data))
.build();
String id = client.getPipeline().getLeader().getUuidString();
@@ -387,12 +387,12 @@ public final class ContainerProtocolCalls {
*/
public static GetSmallFileResponseProto readSmallFile(XceiverClientSpi client,
BlockID blockID, String traceID) throws IOException {
- GetKeyRequestProto.Builder getKey = GetKeyRequestProto
+ GetBlockRequestProto.Builder getBlock = GetBlockRequestProto
.newBuilder()
.setBlockID(blockID.getDatanodeBlockIDProtobuf());
ContainerProtos.GetSmallFileRequestProto getSmallFileRequest =
GetSmallFileRequestProto
- .newBuilder().setKey(getKey)
+ .newBuilder().setBlock(getBlock)
.build();
String id = client.getPipeline().getLeader().getUuidString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
new file mode 100644
index 0000000..0c1d427
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockData.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.helpers;
+
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.client.BlockID;
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.ArrayList;
+
+/**
+ * Helper class to convert Protobuf to Java classes.
+ */
+public class BlockData {
+ private final BlockID blockID;
+ private final Map<String, String> metadata;
+
+ /**
+ * Represent a list of chunks.
+ * In order to reduce memory usage, chunkList is declared as an
+ * {@link Object}.
+ * When #elements == 0, chunkList is null.
+ * When #elements == 1, chunkList refers to the only element.
+ * When #elements > 1, chunkList refers to the list.
+ *
+ * Please note : when we are working with blocks, we don't care what they
+ * point to. So we We don't read chunkinfo nor validate them. It is
+ * responsibility of higher layer like ozone. We just read and write data
+ * from network.
+ */
+ private Object chunkList;
+
+ /**
+ * total size of the key.
+ */
+ private long size;
+
+ /**
+ * Constructs a BlockData Object.
+ *
+ * @param blockID
+ */
+ public BlockData(BlockID blockID) {
+ this.blockID = blockID;
+ this.metadata = new TreeMap<>();
+ this.size = 0;
+ }
+
+ /**
+ * Returns a blockData object from the protobuf data.
+ *
+ * @param data - Protobuf data.
+ * @return - BlockData
+ * @throws IOException
+ */
+ public static BlockData getFromProtoBuf(ContainerProtos.BlockData data) throws
+ IOException {
+ BlockData blockData = new BlockData(
+ BlockID.getFromProtobuf(data.getBlockID()));
+ for (int x = 0; x < data.getMetadataCount(); x++) {
+ blockData.addMetadata(data.getMetadata(x).getKey(),
+ data.getMetadata(x).getValue());
+ }
+ blockData.setChunks(data.getChunksList());
+ if (data.hasSize()) {
+ Preconditions.checkArgument(data.getSize() == blockData.getSize());
+ }
+ return blockData;
+ }
+
+ /**
+ * Returns a Protobuf message from BlockData.
+ * @return Proto Buf Message.
+ */
+ public ContainerProtos.BlockData getProtoBufMessage() {
+ ContainerProtos.BlockData.Builder builder =
+ ContainerProtos.BlockData.newBuilder();
+ builder.setBlockID(this.blockID.getDatanodeBlockIDProtobuf());
+ for (Map.Entry<String, String> entry : metadata.entrySet()) {
+ ContainerProtos.KeyValue.Builder keyValBuilder =
+ ContainerProtos.KeyValue.newBuilder();
+ builder.addMetadata(keyValBuilder.setKey(entry.getKey())
+ .setValue(entry.getValue()).build());
+ }
+ builder.addAllChunks(getChunks());
+ builder.setSize(size);
+ return builder.build();
+ }
+
+ /**
+ * Adds metadata.
+ *
+ * @param key - Key
+ * @param value - Value
+ * @throws IOException
+ */
+ public synchronized void addMetadata(String key, String value) throws
+ IOException {
+ if (this.metadata.containsKey(key)) {
+ throw new IOException("This key already exists. Key " + key);
+ }
+ metadata.put(key, value);
+ }
+
+ public synchronized Map<String, String> getMetadata() {
+ return Collections.unmodifiableMap(this.metadata);
+ }
+
+ /**
+ * Returns value of a key.
+ */
+ public synchronized String getValue(String key) {
+ return metadata.get(key);
+ }
+
+ /**
+ * Deletes a metadata entry from the map.
+ *
+ * @param key - Key
+ */
+ public synchronized void deleteKey(String key) {
+ metadata.remove(key);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<ContainerProtos.ChunkInfo> castChunkList() {
+ return (List<ContainerProtos.ChunkInfo>)chunkList;
+ }
+
+ /**
+ * Returns chunks list.
+ *
+ * @return list of chunkinfo.
+ */
+ public List<ContainerProtos.ChunkInfo> getChunks() {
+ return chunkList == null? Collections.emptyList()
+ : chunkList instanceof ContainerProtos.ChunkInfo?
+ Collections.singletonList((ContainerProtos.ChunkInfo)chunkList)
+ : Collections.unmodifiableList(castChunkList());
+ }
+
+ /**
+ * Adds chinkInfo to the list.
+ */
+ public void addChunk(ContainerProtos.ChunkInfo chunkInfo) {
+ if (chunkList == null) {
+ chunkList = chunkInfo;
+ } else {
+ final List<ContainerProtos.ChunkInfo> list;
+ if (chunkList instanceof ContainerProtos.ChunkInfo) {
+ list = new ArrayList<>(2);
+ list.add((ContainerProtos.ChunkInfo)chunkList);
+ chunkList = list;
+ } else {
+ list = castChunkList();
+ }
+ list.add(chunkInfo);
+ }
+ size += chunkInfo.getLen();
+ }
+
+ /**
+ * removes the chunk.
+ */
+ public boolean removeChunk(ContainerProtos.ChunkInfo chunkInfo) {
+ final boolean removed;
+ if (chunkList instanceof List) {
+ final List<ContainerProtos.ChunkInfo> list = castChunkList();
+ removed = list.remove(chunkInfo);
+ if (list.size() == 1) {
+ chunkList = list.get(0);
+ }
+ } else if (chunkInfo.equals(chunkList)) {
+ chunkList = null;
+ removed = true;
+ } else {
+ removed = false;
+ }
+
+ if (removed) {
+ size -= chunkInfo.getLen();
+ }
+ return removed;
+ }
+
+ /**
+ * Returns container ID.
+ *
+ * @return long.
+ */
+ public long getContainerID() {
+ return blockID.getContainerID();
+ }
+
+ /**
+ * Returns LocalID.
+ * @return long.
+ */
+ public long getLocalID() {
+ return blockID.getLocalID();
+ }
+
+ /**
+ * Return Block ID.
+ * @return BlockID.
+ */
+ public BlockID getBlockID() {
+ return blockID;
+ }
+
+ /**
+ * Sets Chunk list.
+ *
+ * @param chunks - List of chunks.
+ */
+ public void setChunks(List<ContainerProtos.ChunkInfo> chunks) {
+ if (chunks == null) {
+ chunkList = null;
+ size = 0L;
+ } else {
+ final int n = chunks.size();
+ chunkList = n == 0? null: n == 1? chunks.get(0): chunks;
+ size = chunks.parallelStream().mapToLong(
+ ContainerProtos.ChunkInfo::getLen).sum();
+ }
+ }
+
+ /**
+ * Get the total size of chunks allocated for the key.
+ * @return total size of the key.
+ */
+ public long getSize() {
+ return size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
deleted file mode 100644
index ee27021..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.client.BlockID;
-import com.google.common.base.Preconditions;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.ArrayList;
-
-/**
- * Helper class to convert Protobuf to Java classes.
- */
-public class KeyData {
- private final BlockID blockID;
- private final Map<String, String> metadata;
-
- /**
- * Represent a list of chunks.
- * In order to reduce memory usage, chunkList is declared as an
- * {@link Object}.
- * When #elements == 0, chunkList is null.
- * When #elements == 1, chunkList refers to the only element.
- * When #elements > 1, chunkList refers to the list.
- *
- * Please note : when we are working with keys, we don't care what they point
- * to. So we We don't read chunkinfo nor validate them. It is responsibility
- * of higher layer like ozone. We just read and write data from network.
- */
- private Object chunkList;
-
- /**
- * total size of the key.
- */
- private long size;
-
- /**
- * Constructs a KeyData Object.
- *
- * @param blockID
- */
- public KeyData(BlockID blockID) {
- this.blockID = blockID;
- this.metadata = new TreeMap<>();
- this.size = 0;
- }
-
- /**
- * Returns a keyData object from the protobuf data.
- *
- * @param data - Protobuf data.
- * @return - KeyData
- * @throws IOException
- */
- public static KeyData getFromProtoBuf(ContainerProtos.KeyData data) throws
- IOException {
- KeyData keyData = new KeyData(BlockID.getFromProtobuf(data.getBlockID()));
- for (int x = 0; x < data.getMetadataCount(); x++) {
- keyData.addMetadata(data.getMetadata(x).getKey(),
- data.getMetadata(x).getValue());
- }
- keyData.setChunks(data.getChunksList());
- if (data.hasSize()) {
- Preconditions.checkArgument(data.getSize() == keyData.getSize());
- }
- return keyData;
- }
-
- /**
- * Returns a Protobuf message from KeyData.
- * @return Proto Buf Message.
- */
- public ContainerProtos.KeyData getProtoBufMessage() {
- ContainerProtos.KeyData.Builder builder =
- ContainerProtos.KeyData.newBuilder();
- builder.setBlockID(this.blockID.getDatanodeBlockIDProtobuf());
- for (Map.Entry<String, String> entry : metadata.entrySet()) {
- ContainerProtos.KeyValue.Builder keyValBuilder =
- ContainerProtos.KeyValue.newBuilder();
- builder.addMetadata(keyValBuilder.setKey(entry.getKey())
- .setValue(entry.getValue()).build());
- }
- builder.addAllChunks(getChunks());
- builder.setSize(size);
- return builder.build();
- }
-
- /**
- * Adds metadata.
- *
- * @param key - Key
- * @param value - Value
- * @throws IOException
- */
- public synchronized void addMetadata(String key, String value) throws
- IOException {
- if (this.metadata.containsKey(key)) {
- throw new IOException("This key already exists. Key " + key);
- }
- metadata.put(key, value);
- }
-
- public synchronized Map<String, String> getMetadata() {
- return Collections.unmodifiableMap(this.metadata);
- }
-
- /**
- * Returns value of a key.
- */
- public synchronized String getValue(String key) {
- return metadata.get(key);
- }
-
- /**
- * Deletes a metadata entry from the map.
- *
- * @param key - Key
- */
- public synchronized void deleteKey(String key) {
- metadata.remove(key);
- }
-
- @SuppressWarnings("unchecked")
- private List<ContainerProtos.ChunkInfo> castChunkList() {
- return (List<ContainerProtos.ChunkInfo>)chunkList;
- }
-
- /**
- * Returns chunks list.
- *
- * @return list of chunkinfo.
- */
- public List<ContainerProtos.ChunkInfo> getChunks() {
- return chunkList == null? Collections.emptyList()
- : chunkList instanceof ContainerProtos.ChunkInfo?
- Collections.singletonList((ContainerProtos.ChunkInfo)chunkList)
- : Collections.unmodifiableList(castChunkList());
- }
-
- /**
- * Adds chinkInfo to the list.
- */
- public void addChunk(ContainerProtos.ChunkInfo chunkInfo) {
- if (chunkList == null) {
- chunkList = chunkInfo;
- } else {
- final List<ContainerProtos.ChunkInfo> list;
- if (chunkList instanceof ContainerProtos.ChunkInfo) {
- list = new ArrayList<>(2);
- list.add((ContainerProtos.ChunkInfo)chunkList);
- chunkList = list;
- } else {
- list = castChunkList();
- }
- list.add(chunkInfo);
- }
- size += chunkInfo.getLen();
- }
-
- /**
- * removes the chunk.
- */
- public boolean removeChunk(ContainerProtos.ChunkInfo chunkInfo) {
- final boolean removed;
- if (chunkList instanceof List) {
- final List<ContainerProtos.ChunkInfo> list = castChunkList();
- removed = list.remove(chunkInfo);
- if (list.size() == 1) {
- chunkList = list.get(0);
- }
- } else if (chunkInfo.equals(chunkList)) {
- chunkList = null;
- removed = true;
- } else {
- removed = false;
- }
-
- if (removed) {
- size -= chunkInfo.getLen();
- }
- return removed;
- }
-
- /**
- * Returns container ID.
- *
- * @return long.
- */
- public long getContainerID() {
- return blockID.getContainerID();
- }
-
- /**
- * Returns LocalID.
- * @return long.
- */
- public long getLocalID() {
- return blockID.getLocalID();
- }
-
- /**
- * Return Block ID.
- * @return BlockID.
- */
- public BlockID getBlockID() {
- return blockID;
- }
-
- /**
- * Sets Chunk list.
- *
- * @param chunks - List of chunks.
- */
- public void setChunks(List<ContainerProtos.ChunkInfo> chunks) {
- if (chunks == null) {
- chunkList = null;
- size = 0L;
- } else {
- final int n = chunks.size();
- chunkList = n == 0? null: n == 1? chunks.get(0): chunks;
- size = chunks.parallelStream().mapToLong(
- ContainerProtos.ChunkInfo::getLen).sum();
- }
- }
-
- /**
- * Get the total size of chunks allocated for the key.
- * @return total size of the key.
- */
- public long getSize() {
- return size;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index ba0d2d4..7be8a62 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -49,13 +49,13 @@ package hadoop.hdds.datanode;
* 5. ListContainer - Returns the list of containers on this
* datanode. This will be used by tests and tools.
*
- * 6. PutKey - Given a valid container, creates a key.
+ * 6. PutBlock - Given a valid container, creates a block.
*
- * 7. GetKey - Allows user to read the metadata of a Key.
+ * 7. GetBlock - Allows user to read the metadata of a block.
*
- * 8. DeleteKey - Deletes a given key.
+ * 8. DeleteBlock - Deletes a given block.
*
- * 9. ListKey - Returns a list of keys that are present inside
+ * 9. ListBlock - Returns a list of blocks that are present inside
* a given container.
*
* 10. ReadChunk - Allows us to read a chunk.
@@ -64,13 +64,13 @@ package hadoop.hdds.datanode;
*
* 12. WriteChunk - Allows us to write a chunk
*
- * 13. ListChunk - Given a Container/Key returns the list of Chunks.
+ * 13. ListChunk - Given a Container/Block returns the list of Chunks.
*
* 14. CompactChunk - Re-writes a chunk based on Offsets.
*
- * 15. PutSmallFile - A single RPC that combines both putKey and WriteChunk.
+ * 15. PutSmallFile - A single RPC that combines both putBlock and WriteChunk.
*
- * 16. GetSmallFile - A single RPC that combines both getKey and ReadChunk.
+ * 16. GetSmallFile - A single RPC that combines both getBlock and ReadChunk.
*
* 17. CloseContainer - Closes an open container and makes it immutable.
*
@@ -84,10 +84,10 @@ enum Type {
DeleteContainer = 4;
ListContainer = 5;
- PutKey = 6;
- GetKey = 7;
- DeleteKey = 8;
- ListKey = 9;
+ PutBlock = 6;
+ GetBlock = 7;
+ DeleteBlock = 8;
+ ListBlock = 9;
ReadChunk = 10;
DeleteChunk = 11;
@@ -95,7 +95,7 @@ enum Type {
ListChunk = 13;
CompactChunk = 14;
- /** Combines Key and Chunk Operation into Single RPC. */
+ /** Combines Block and Chunk Operation into Single RPC. */
PutSmallFile = 15;
GetSmallFile = 16;
CloseContainer = 17;
@@ -115,7 +115,7 @@ enum Result {
CONTAINER_NOT_FOUND = 9;
IO_EXCEPTION = 10;
UNABLE_TO_READ_METADATA_DB = 11;
- NO_SUCH_KEY = 12;
+ NO_SUCH_BLOCK = 12;
OVERWRITE_FLAG_REQUIRED = 13;
UNABLE_TO_FIND_DATA_DIR = 14;
INVALID_WRITE_SIZE = 15;
@@ -185,10 +185,10 @@ message ContainerCommandRequestProto {
optional ListContainerRequestProto listContainer = 9;
optional CloseContainerRequestProto closeContainer = 10;
- optional PutKeyRequestProto putKey = 11;
- optional GetKeyRequestProto getKey = 12;
- optional DeleteKeyRequestProto deleteKey = 13;
- optional ListKeyRequestProto listKey = 14;
+ optional PutBlockRequestProto putBlock = 11;
+ optional GetBlockRequestProto getBlock = 12;
+ optional DeleteBlockRequestProto deleteBlock = 13;
+ optional ListBlockRequestProto listBlock = 14;
optional ReadChunkRequestProto readChunk = 15;
optional WriteChunkRequestProto writeChunk = 16;
@@ -215,10 +215,10 @@ message ContainerCommandResponseProto {
optional ListContainerResponseProto listContainer = 9;
optional CloseContainerResponseProto closeContainer = 10;
- optional PutKeyResponseProto putKey = 11;
- optional GetKeyResponseProto getKey = 12;
- optional DeleteKeyResponseProto deleteKey = 13;
- optional ListKeyResponseProto listKey = 14;
+ optional PutBlockResponseProto putBlock = 11;
+ optional GetBlockResponseProto getBlock = 12;
+ optional DeleteBlockResponseProto deleteBlock = 13;
+ optional ListBlockResponseProto listBlock = 14;
optional WriteChunkResponseProto writeChunk = 15;
optional ReadChunkResponseProto readChunk = 16;
@@ -294,7 +294,7 @@ message CloseContainerResponseProto {
optional int64 containerID = 2;
}
-message KeyData {
+message BlockData {
required DatanodeBlockID blockID = 1;
optional int64 flags = 2; // for future use.
repeated KeyValue metadata = 3;
@@ -302,25 +302,25 @@ message KeyData {
optional int64 size = 5;
}
-// Key Messages.
-message PutKeyRequestProto {
- required KeyData keyData = 1;
+// Block Messages.
+message PutBlockRequestProto {
+ required BlockData blockData = 1;
}
-message PutKeyResponseProto {
+message PutBlockResponseProto {
required GetCommittedBlockLengthResponseProto committedBlockLength = 1;
}
-message GetKeyRequestProto {
+message GetBlockRequestProto {
required DatanodeBlockID blockID = 1;
}
-message GetKeyResponseProto {
- required KeyData keyData = 1;
+message GetBlockResponseProto {
+ required BlockData blockData = 1;
}
-message DeleteKeyRequestProto {
+message DeleteBlockRequestProto {
required DatanodeBlockID blockID = 1;
}
@@ -333,17 +333,17 @@ message GetCommittedBlockLengthResponseProto {
required int64 blockLength = 2;
}
-message DeleteKeyResponseProto {
+message DeleteBlockResponseProto {
}
-message ListKeyRequestProto {
+message ListBlockRequestProto {
optional int64 startLocalID = 2;
required uint32 count = 3;
}
-message ListKeyResponseProto {
- repeated KeyData keyData = 1;
+message ListBlockResponseProto {
+ repeated BlockData blockData = 1;
}
// Chunk Operations
@@ -401,11 +401,11 @@ message ListChunkResponseProto {
repeated ChunkInfo chunkData = 1;
}
-/** For small file access combines write chunk and putKey into a single
+/** For small file access combines write chunk and putBlock into a single
RPC */
message PutSmallFileRequestProto {
- required PutKeyRequestProto key = 1;
+ required PutBlockRequestProto block = 1;
required ChunkInfo chunkInfo = 2;
required bytes data = 3;
}
@@ -416,7 +416,7 @@ message PutSmallFileResponseProto {
}
message GetSmallFileRequestProto {
- required GetKeyRequestProto key = 1;
+ required GetBlockRequestProto block = 1;
}
message GetSmallFileResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java
index 1ef3d0d..b736eb5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java
@@ -22,7 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import java.util.ArrayList;
import java.util.Collections;
@@ -33,9 +33,9 @@ import java.util.concurrent.ConcurrentMap;
import java.util.function.Function;
/**
- * Map: containerId -> (localId -> {@link KeyData}).
+ * Map: containerId {@literal ->} (localId {@literal ->} {@link BlockData}).
* The outer container map does not entail locking for a better performance.
- * The inner {@link KeyDataMap} is synchronized.
+ * The inner {@link BlockDataMap} is synchronized.
*
* This class will maintain list of open keys per container when closeContainer
* command comes, it should autocommit all open keys of a open container before
@@ -43,16 +43,16 @@ import java.util.function.Function;
*/
public class OpenContainerBlockMap {
/**
- * Map: localId -> KeyData.
+ * Map: localId {@literal ->} BlockData.
*
* In order to support {@link #getAll()}, the update operations are
* synchronized.
*/
- static class KeyDataMap {
- private final ConcurrentMap<Long, KeyData> blocks =
+ static class BlockDataMap {
+ private final ConcurrentMap<Long, BlockData> blocks =
new ConcurrentHashMap<>();
- KeyData get(long localId) {
+ BlockData get(long localId) {
return blocks.get(localId);
}
@@ -61,12 +61,12 @@ public class OpenContainerBlockMap {
return blocks.size();
}
- synchronized KeyData computeIfAbsent(
- long localId, Function<Long, KeyData> f) {
+ synchronized BlockData computeIfAbsent(
+ long localId, Function<Long, BlockData> f) {
return blocks.computeIfAbsent(localId, f);
}
- synchronized List<KeyData> getAll() {
+ synchronized List<BlockData> getAll() {
return new ArrayList<>(blocks.values());
}
}
@@ -79,7 +79,7 @@ public class OpenContainerBlockMap {
*
* For now, we will track all open blocks of a container in the blockMap.
*/
- private final ConcurrentMap<Long, KeyDataMap> containers =
+ private final ConcurrentMap<Long, BlockDataMap> containers =
new ConcurrentHashMap<>();
/**
@@ -94,9 +94,9 @@ public class OpenContainerBlockMap {
public void addChunk(BlockID blockID, ChunkInfo info) {
Preconditions.checkNotNull(info);
- containers.computeIfAbsent(blockID.getContainerID(), id -> new KeyDataMap())
- .computeIfAbsent(blockID.getLocalID(), id -> new KeyData(blockID))
- .addChunk(info);
+ containers.computeIfAbsent(blockID.getContainerID(),
+ id -> new BlockDataMap()).computeIfAbsent(blockID.getLocalID(),
+ id -> new BlockData(blockID)).addChunk(info);
}
/**
@@ -113,21 +113,21 @@ public class OpenContainerBlockMap {
}
/**
- * Returns the list of open to the openContainerBlockMap.
+ * Returns the list of open blocks to the openContainerBlockMap.
* @param containerId container id
- * @return List of open Keys(blocks)
+ * @return List of open blocks
*/
- public List<KeyData> getOpenKeys(long containerId) {
+ public List<BlockData> getOpenBlocks(long containerId) {
return Optional.ofNullable(containers.get(containerId))
- .map(KeyDataMap::getAll)
+ .map(BlockDataMap::getAll)
.orElseGet(Collections::emptyList);
}
/**
* removes the block from the block map.
- * @param blockID
+ * @param blockID - block ID
*/
- public void removeFromKeyMap(BlockID blockID) {
+ public void removeFromBlockMap(BlockID blockID) {
Preconditions.checkNotNull(blockID);
containers.computeIfPresent(blockID.getContainerID(), (containerId, blocks)
-> blocks.removeAndGetSize(blockID.getLocalID()) == 0? null: blocks);
@@ -136,16 +136,16 @@ public class OpenContainerBlockMap {
/**
* Returns true if the block exists in the map, false otherwise.
*
- * @param blockID
+ * @param blockID - Block ID.
* @return True, if it exists, false otherwise
*/
public boolean checkIfBlockExists(BlockID blockID) {
- KeyDataMap keyDataMap = containers.get(blockID.getContainerID());
+ BlockDataMap keyDataMap = containers.get(blockID.getContainerID());
return keyDataMap != null && keyDataMap.get(blockID.getLocalID()) != null;
}
@VisibleForTesting
- KeyDataMap getKeyDataMap(long containerId) {
+ BlockDataMap getBlockDataMap(long containerId) {
return containers.get(containerId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index b0d4cbc..430b0ef 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.container.common.helpers
.DeletedContainerBlocksSummary;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.statemachine
.EndpointStateMachine;
@@ -199,7 +199,7 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
}
int newDeletionBlocks = 0;
- MetadataStore containerDB = KeyUtils.getDB(containerData, conf);
+ MetadataStore containerDB = BlockUtils.getDB(containerData, conf);
for (Long blk : delTX.getLocalIDList()) {
BatchOperation batch = new BatchOperation();
byte[] blkBytes = Longs.toByteArray(blk);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index b84db66..a7bef86 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -103,10 +103,10 @@ import java.util.stream.Collectors;
* implementation. For example, synchronization between writeChunk and
* createContainer in {@link ContainerStateMachine}.
*
- * PutKey is synchronized with WriteChunk operations, PutKey for a block is
- * executed only after all the WriteChunk preceding the PutKey have finished.
+ * PutBlock is synchronized with WriteChunk operations, PutBlock for a block is
+ * executed only after all the WriteChunk preceding the PutBlock have finished.
*
- * CloseContainer is synchronized with WriteChunk and PutKey operations,
+ * CloseContainer is synchronized with WriteChunk and PutBlock operations,
* CloseContainer for a container is processed after all the preceding write
* operations for the container have finished.
* */
@@ -443,7 +443,7 @@ public class ContainerStateMachine extends BaseStateMachine {
/**
* This class maintains maps and provide utilities to enforce synchronization
- * among createContainer, writeChunk, putKey and closeContainer.
+ * among createContainer, writeChunk, putBlock and closeContainer.
*/
private class StateMachineHelper {
@@ -453,7 +453,7 @@ public class ContainerStateMachine extends BaseStateMachine {
private final ConcurrentHashMap<Long, CommitChunkFutureMap>
block2ChunkMap;
- // Map for putKey futures
+ // Map for putBlock futures
private final ConcurrentHashMap<Long, CompletableFuture<Message>>
blockCommitMap;
@@ -505,11 +505,11 @@ public class ContainerStateMachine extends BaseStateMachine {
// The following section handles applyTransaction transactions
// on a container
- private CompletableFuture<Message> handlePutKey(
+ private CompletableFuture<Message> handlePutBlock(
ContainerCommandRequestProto requestProto) {
List<CompletableFuture<Message>> futureList = new ArrayList<>();
long localId =
- requestProto.getPutKey().getKeyData().getBlockID().getLocalID();
+ requestProto.getPutBlock().getBlockData().getBlockID().getLocalID();
// Need not wait for create container future here as it has already
// finished.
if (block2ChunkMap.get(localId) != null) {
@@ -518,18 +518,18 @@ public class ContainerStateMachine extends BaseStateMachine {
CompletableFuture<Message> effectiveFuture =
runCommandAfterFutures(futureList, requestProto);
- CompletableFuture<Message> putKeyFuture =
+ CompletableFuture<Message> putBlockFuture =
effectiveFuture.thenApply(message -> {
blockCommitMap.remove(localId);
return message;
});
- blockCommitMap.put(localId, putKeyFuture);
- return putKeyFuture;
+ blockCommitMap.put(localId, putBlockFuture);
+ return putBlockFuture;
}
// Close Container should be executed only if all pending WriteType
// container cmds get executed. Transactions which can return a future
- // are WriteChunk and PutKey.
+ // are WriteChunk and PutBlock.
private CompletableFuture<Message> handleCloseContainer(
ContainerCommandRequestProto requestProto) {
List<CompletableFuture<Message>> futureList = new ArrayList<>();
@@ -539,7 +539,7 @@ public class ContainerStateMachine extends BaseStateMachine {
block2ChunkMap.values().forEach(b -> futureList.addAll(b.getAll()));
futureList.addAll(blockCommitMap.values());
- // There are pending write Chunk/PutKey type requests
+ // There are pending write Chunk/PutBlock type requests
// Queue this closeContainer request behind all these requests
CompletableFuture<Message> closeContainerFuture =
runCommandAfterFutures(futureList, requestProto);
@@ -615,8 +615,8 @@ public class ContainerStateMachine extends BaseStateMachine {
return handleChunkCommit(requestProto, index);
case CloseContainer:
return handleCloseContainer(requestProto);
- case PutKey:
- return handlePutKey(requestProto);
+ case PutBlock:
+ return handlePutBlock(requestProto);
case CreateContainer:
return handleCreateContainer(requestProto);
default:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueBlockIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueBlockIterator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueBlockIterator.java
index f800223..535af29 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueBlockIterator.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueBlockIterator.java
@@ -21,12 +21,12 @@ package org.apache.hadoop.ozone.container.keyvalue;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.interfaces.BlockIterator;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.utils.MetaStoreIterator;
import org.apache.hadoop.utils.MetadataKeyFilters;
@@ -48,7 +48,7 @@ import java.util.NoSuchElementException;
* {@link MetadataKeyFilters#getNormalKeyFilter()}
*/
@InterfaceAudience.Public
-public class KeyValueBlockIterator implements BlockIterator<KeyData> {
+public class KeyValueBlockIterator implements BlockIterator<BlockData> {
private static final Logger LOG = LoggerFactory.getLogger(
KeyValueBlockIterator.class);
@@ -57,7 +57,7 @@ public class KeyValueBlockIterator implements BlockIterator<KeyData> {
private static KeyPrefixFilter defaultBlockFilter = MetadataKeyFilters
.getNormalKeyFilter();
private KeyPrefixFilter blockFilter;
- private KeyData nextBlock;
+ private BlockData nextBlock;
private long containerId;
/**
@@ -91,7 +91,7 @@ public class KeyValueBlockIterator implements BlockIterator<KeyData> {
containerData;
keyValueContainerData.setDbFile(KeyValueContainerLocationUtil
.getContainerDBFile(metdataPath, containerId));
- MetadataStore metadataStore = KeyUtils.getDB(keyValueContainerData, new
+ MetadataStore metadataStore = BlockUtils.getDB(keyValueContainerData, new
OzoneConfiguration());
blockIterator = metadataStore.iterator();
blockFilter = filter;
@@ -103,9 +103,9 @@ public class KeyValueBlockIterator implements BlockIterator<KeyData> {
* @throws IOException
*/
@Override
- public KeyData nextBlock() throws IOException, NoSuchElementException {
+ public BlockData nextBlock() throws IOException, NoSuchElementException {
if (nextBlock != null) {
- KeyData currentBlock = nextBlock;
+ BlockData currentBlock = nextBlock;
nextBlock = null;
return currentBlock;
}
@@ -124,7 +124,7 @@ public class KeyValueBlockIterator implements BlockIterator<KeyData> {
if (blockIterator.hasNext()) {
KeyValue block = blockIterator.next();
if (blockFilter.filterKey(null, block.getKey(), null)) {
- nextBlock = KeyUtils.getKeyData(block.getValue());
+ nextBlock = BlockUtils.getBlockData(block.getValue());
LOG.trace("Block matching with filter found: blockID is : {} for " +
"containerID {}", nextBlock.getLocalID(), containerId);
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index 0870c76..09d4054 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers
.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
@@ -293,7 +293,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
// It is ok if this operation takes a bit of time.
// Close container is not expected to be instantaneous.
try {
- MetadataStore db = KeyUtils.getDB(containerData, config);
+ MetadataStore db = BlockUtils.getDB(containerData, config);
db.compactDB();
} catch (StorageContainerException ex) {
throw ex;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 5acecb4..5be6e28 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -48,10 +48,10 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.OpenContainerBlockMap;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -62,13 +62,13 @@ import org.apache.hadoop.ozone.container.common.volume
.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.SmallFileUtils;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
-import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
+import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background
.BlockDeletingService;
import org.apache.hadoop.util.AutoCloseableLock;
@@ -117,7 +117,7 @@ public class KeyValueHandler extends Handler {
KeyValueHandler.class);
private final ContainerType containerType;
- private final KeyManager keyManager;
+ private final BlockManager blockManager;
private final ChunkManager chunkManager;
private final BlockDeletingService blockDeletingService;
private final VolumeChoosingPolicy volumeChoosingPolicy;
@@ -129,7 +129,7 @@ public class KeyValueHandler extends Handler {
VolumeSet volSet, ContainerMetrics metrics) {
super(config, contSet, volSet, metrics);
containerType = ContainerType.KeyValueContainer;
- keyManager = new KeyManagerImpl(config);
+ blockManager = new BlockManagerImpl(config);
chunkManager = new ChunkManagerImpl();
long svcInterval = config
.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
@@ -187,13 +187,13 @@ public class KeyValueHandler extends Handler {
return handleUnsupportedOp(request);
case CloseContainer:
return handleCloseContainer(request, kvContainer);
- case PutKey:
- return handlePutKey(request, kvContainer);
- case GetKey:
- return handleGetKey(request, kvContainer);
- case DeleteKey:
- return handleDeleteKey(request, kvContainer);
- case ListKey:
+ case PutBlock:
+ return handlePutBlock(request, kvContainer);
+ case GetBlock:
+ return handleGetBlock(request, kvContainer);
+ case DeleteBlock:
+ return handleDeleteBlock(request, kvContainer);
+ case ListBlock:
return handleUnsupportedOp(request);
case ReadChunk:
return handleReadChunk(request, kvContainer);
@@ -222,8 +222,8 @@ public class KeyValueHandler extends Handler {
}
@VisibleForTesting
- public KeyManager getKeyManager() {
- return this.keyManager;
+ public BlockManager getBlockManager() {
+ return this.blockManager;
}
/**
@@ -413,7 +413,7 @@ public class KeyValueHandler extends Handler {
// remove the container from open block map once, all the blocks
// have been committed and the container is closed
kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSING);
- commitPendingKeys(kvContainer);
+ commitPendingBlocks(kvContainer);
kvContainer.close();
// make sure the the container open keys from BlockMap gets removed
openContainerBlockMap.removeContainer(kvData.getContainerID());
@@ -429,13 +429,13 @@ public class KeyValueHandler extends Handler {
}
/**
- * Handle Put Key operation. Calls KeyManager to process the request.
+ * Handle Put Block operation. Calls BlockManager to process the request.
*/
- ContainerCommandResponseProto handlePutKey(
+ ContainerCommandResponseProto handlePutBlock(
ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
long blockLength;
- if (!request.hasPutKey()) {
+ if (!request.hasPutBlock()) {
LOG.debug("Malformed Put Key request. trace ID: {}",
request.getTraceID());
return ContainerUtils.malformedRequest(request);
@@ -444,11 +444,11 @@ public class KeyValueHandler extends Handler {
try {
checkContainerOpen(kvContainer);
- KeyData keyData = KeyData.getFromProtoBuf(
- request.getPutKey().getKeyData());
- long numBytes = keyData.getProtoBufMessage().toByteArray().length;
- blockLength = commitKey(keyData, kvContainer);
- metrics.incContainerBytesStats(Type.PutKey, numBytes);
+ BlockData blockData = BlockData.getFromProtoBuf(
+ request.getPutBlock().getBlockData());
+ long numBytes = blockData.getProtoBufMessage().toByteArray().length;
+ blockLength = commitKey(blockData, kvContainer);
+ metrics.incContainerBytesStats(Type.PutBlock, numBytes);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
} catch (IOException ex) {
@@ -457,46 +457,46 @@ public class KeyValueHandler extends Handler {
request);
}
- return KeyUtils.putKeyResponseSuccess(request, blockLength);
+ return BlockUtils.putBlockResponseSuccess(request, blockLength);
}
- private void commitPendingKeys(KeyValueContainer kvContainer)
+ private void commitPendingBlocks(KeyValueContainer kvContainer)
throws IOException {
long containerId = kvContainer.getContainerData().getContainerID();
- List<KeyData> pendingKeys =
- this.openContainerBlockMap.getOpenKeys(containerId);
- for(KeyData keyData : pendingKeys) {
- commitKey(keyData, kvContainer);
+ List<BlockData> pendingBlocks =
+ this.openContainerBlockMap.getOpenBlocks(containerId);
+ for(BlockData blockData : pendingBlocks) {
+ commitKey(blockData, kvContainer);
}
}
- private long commitKey(KeyData keyData, KeyValueContainer kvContainer)
+ private long commitKey(BlockData blockData, KeyValueContainer kvContainer)
throws IOException {
- Preconditions.checkNotNull(keyData);
- long length = keyManager.putKey(kvContainer, keyData);
+ Preconditions.checkNotNull(blockData);
+ long length = blockManager.putBlock(kvContainer, blockData);
//update the open key Map in containerManager
- this.openContainerBlockMap.removeFromKeyMap(keyData.getBlockID());
+ this.openContainerBlockMap.removeFromBlockMap(blockData.getBlockID());
return length;
}
/**
- * Handle Get Key operation. Calls KeyManager to process the request.
+ * Handle Get Block operation. Calls BlockManager to process the request.
*/
- ContainerCommandResponseProto handleGetKey(
+ ContainerCommandResponseProto handleGetBlock(
ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
- if (!request.hasGetKey()) {
+ if (!request.hasGetBlock()) {
LOG.debug("Malformed Get Key request. trace ID: {}",
request.getTraceID());
return ContainerUtils.malformedRequest(request);
}
- KeyData responseData;
+ BlockData responseData;
try {
BlockID blockID = BlockID.getFromProtobuf(
- request.getGetKey().getBlockID());
- responseData = keyManager.getKey(kvContainer, blockID);
+ request.getGetBlock().getBlockID());
+ responseData = blockManager.getBlock(kvContainer, blockID);
long numBytes = responseData.getProtoBufMessage().toByteArray().length;
- metrics.incContainerBytesStats(Type.GetKey, numBytes);
+ metrics.incContainerBytesStats(Type.GetBlock, numBytes);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -506,12 +506,12 @@ public class KeyValueHandler extends Handler {
request);
}
- return KeyUtils.getKeyDataResponse(request, responseData);
+ return BlockUtils.getBlockDataResponse(request, responseData);
}
/**
* Handles GetCommittedBlockLength operation.
- * Calls KeyManager to process the request.
+ * Calls BlockManager to process the request.
*/
ContainerCommandResponseProto handleGetCommittedBlockLength(
ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
@@ -530,7 +530,7 @@ public class KeyValueHandler extends Handler {
String msg = "Block " + blockID + " is not committed yet.";
throw new StorageContainerException(msg, BLOCK_NOT_COMMITTED);
}
- blockLength = keyManager.getCommittedBlockLength(kvContainer, blockID);
+ blockLength = blockManager.getCommittedBlockLength(kvContainer, blockID);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
} catch (IOException ex) {
@@ -539,16 +539,16 @@ public class KeyValueHandler extends Handler {
IO_EXCEPTION), request);
}
- return KeyUtils.getBlockLengthResponse(request, blockLength);
+ return BlockUtils.getBlockLengthResponse(request, blockLength);
}
/**
- * Handle Delete Key operation. Calls KeyManager to process the request.
+ * Handle Delete Block operation. Calls BlockManager to process the request.
*/
- ContainerCommandResponseProto handleDeleteKey(
+ ContainerCommandResponseProto handleDeleteBlock(
ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
- if (!request.hasDeleteKey()) {
+ if (!request.hasDeleteBlock()) {
LOG.debug("Malformed Delete Key request. trace ID: {}",
request.getTraceID());
return ContainerUtils.malformedRequest(request);
@@ -558,9 +558,9 @@ public class KeyValueHandler extends Handler {
checkContainerOpen(kvContainer);
BlockID blockID = BlockID.getFromProtobuf(
- request.getDeleteKey().getBlockID());
+ request.getDeleteBlock().getBlockID());
- keyManager.deleteKey(kvContainer, blockID);
+ blockManager.deleteBlock(kvContainer, blockID);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
} catch (IOException ex) {
@@ -569,7 +569,7 @@ public class KeyValueHandler extends Handler {
request);
}
- return KeyUtils.getKeyResponseSuccess(request);
+ return BlockUtils.getBlockResponseSuccess(request);
}
/**
@@ -698,7 +698,7 @@ public class KeyValueHandler extends Handler {
/**
* Handle Put Small File operation. Writes the chunk and associated key
- * using a single RPC. Calls KeyManager and ChunkManager to process the
+ * using a single RPC. Calls BlockManager and ChunkManager to process the
* request.
*/
ContainerCommandResponseProto handlePutSmallFile(
@@ -715,11 +715,11 @@ public class KeyValueHandler extends Handler {
try {
checkContainerOpen(kvContainer);
- BlockID blockID = BlockID.getFromProtobuf(putSmallFileReq.getKey()
- .getKeyData().getBlockID());
- KeyData keyData = KeyData.getFromProtoBuf(
- putSmallFileReq.getKey().getKeyData());
- Preconditions.checkNotNull(keyData);
+ BlockID blockID = BlockID.getFromProtobuf(putSmallFileReq.getBlock()
+ .getBlockData().getBlockID());
+ BlockData blockData = BlockData.getFromProtoBuf(
+ putSmallFileReq.getBlock().getBlockData());
+ Preconditions.checkNotNull(blockData);
ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(
putSmallFileReq.getChunkInfo());
@@ -732,8 +732,8 @@ public class KeyValueHandler extends Handler {
List<ContainerProtos.ChunkInfo> chunks = new LinkedList<>();
chunks.add(chunkInfo.getProtoBufMessage());
- keyData.setChunks(chunks);
- keyManager.putKey(kvContainer, keyData);
+ blockData.setChunks(chunks);
+ blockManager.putBlock(kvContainer, blockData);
metrics.incContainerBytesStats(Type.PutSmallFile, data.length);
} catch (StorageContainerException ex) {
@@ -749,7 +749,7 @@ public class KeyValueHandler extends Handler {
/**
* Handle Get Small File operation. Gets a data stream using a key. This
- * helps in reducing the RPC overhead for small files. Calls KeyManager and
+ * helps in reducing the RPC overhead for small files. Calls BlockManager and
* ChunkManager to process the request.
*/
ContainerCommandResponseProto handleGetSmallFile(
@@ -764,9 +764,9 @@ public class KeyValueHandler extends Handler {
GetSmallFileRequestProto getSmallFileReq = request.getGetSmallFile();
try {
- BlockID blockID = BlockID.getFromProtobuf(getSmallFileReq.getKey()
+ BlockID blockID = BlockID.getFromProtobuf(getSmallFileReq.getBlock()
.getBlockID());
- KeyData responseData = keyManager.getKey(kvContainer, blockID);
+ BlockData responseData = blockManager.getBlock(kvContainer, blockID);
ContainerProtos.ChunkInfo chunkInfo = null;
ByteString dataBuf = ByteString.EMPTY;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
new file mode 100644
index 0000000..f5cc847
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.helpers;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .GetBlockResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
+ GetCommittedBlockLengthResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
+ PutBlockResponseProto;
+import org.apache.hadoop.hdds.scm.container.common.helpers
+ .StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
+import org.apache.hadoop.utils.MetadataStore;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .Result.NO_SUCH_BLOCK;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .Result.UNABLE_TO_READ_METADATA_DB;
+
+/**
+ * Utils functions to help block functions.
+ */
+public final class BlockUtils {
+
+ /** Never constructed. **/
+ private BlockUtils() {
+
+ }
+ /**
+ * Get a DB handler for a given container.
+ * If the handler doesn't exist in cache yet, first create one and
+ * add into cache. This function is called with containerManager
+ * ReadLock held.
+ *
+ * @param containerData containerData.
+ * @param conf configuration.
+ * @return MetadataStore handle.
+ * @throws StorageContainerException
+ */
+ public static MetadataStore getDB(KeyValueContainerData containerData,
+ Configuration conf) throws
+ StorageContainerException {
+ Preconditions.checkNotNull(containerData);
+ ContainerCache cache = ContainerCache.getInstance(conf);
+ Preconditions.checkNotNull(cache);
+ Preconditions.checkNotNull(containerData.getDbFile());
+ try {
+ return cache.getDB(containerData.getContainerID(), containerData
+ .getContainerDBType(), containerData.getDbFile().getAbsolutePath());
+ } catch (IOException ex) {
+ String message = String.format("Error opening DB. Container:%s " +
+ "ContainerPath:%s", containerData.getContainerID(), containerData
+ .getDbFile().getPath());
+ throw new StorageContainerException(message, UNABLE_TO_READ_METADATA_DB);
+ }
+ }
+ /**
+ * Remove a DB handler from cache.
+ *
+ * @param container - Container data.
+ * @param conf - Configuration.
+ */
+ public static void removeDB(KeyValueContainerData container, Configuration
+ conf) {
+ Preconditions.checkNotNull(container);
+ ContainerCache cache = ContainerCache.getInstance(conf);
+ Preconditions.checkNotNull(cache);
+ cache.removeDB(container.getContainerID());
+ }
+
+ /**
+ * Shutdown all DB Handles.
+ *
+ * @param cache - Cache for DB Handles.
+ */
+ @SuppressWarnings("unchecked")
+ public static void shutdownCache(ContainerCache cache) {
+ cache.shutdownCache();
+ }
+
+ /**
+ * Parses the {@link BlockData} from a bytes array.
+ *
+ * @param bytes Block data in bytes.
+ * @return Block data.
+ * @throws IOException if the bytes array is malformed or invalid.
+ */
+ public static BlockData getBlockData(byte[] bytes) throws IOException {
+ try {
+ ContainerProtos.BlockData blockData = ContainerProtos.BlockData.parseFrom(
+ bytes);
+ BlockData data = BlockData.getFromProtoBuf(blockData);
+ return data;
+ } catch (IOException e) {
+ throw new StorageContainerException("Failed to parse block data from " +
+ "the bytes array.", NO_SUCH_BLOCK);
+ }
+ }
+
+ /**
+ * Returns putBlock response success.
+ * @param msg - Request.
+ * @return Response.
+ */
+ public static ContainerCommandResponseProto putBlockResponseSuccess(
+ ContainerCommandRequestProto msg, long blockLength) {
+ GetCommittedBlockLengthResponseProto.Builder
+ committedBlockLengthResponseBuilder =
+ getCommittedBlockLengthResponseBuilder(blockLength,
+ msg.getPutBlock().getBlockData().getBlockID());
+ PutBlockResponseProto.Builder putKeyResponse =
+ PutBlockResponseProto.newBuilder();
+ putKeyResponse
+ .setCommittedBlockLength(committedBlockLengthResponseBuilder);
+ ContainerProtos.ContainerCommandResponseProto.Builder builder =
+ ContainerUtils.getSuccessResponseBuilder(msg);
+ builder.setPutBlock(putKeyResponse);
+ return builder.build();
+ }
+ /**
+ * Returns successful blockResponse.
+ * @param msg - Request.
+ * @return Response.
+ */
+ public static ContainerCommandResponseProto getBlockResponseSuccess(
+ ContainerCommandRequestProto msg) {
+ return ContainerUtils.getSuccessResponse(msg);
+ }
+
+
+ public static ContainerCommandResponseProto getBlockDataResponse(
+ ContainerCommandRequestProto msg, BlockData data) {
+ GetBlockResponseProto.Builder getBlock = ContainerProtos
+ .GetBlockResponseProto
+ .newBuilder();
+ getBlock.setBlockData(data.getProtoBufMessage());
+ ContainerProtos.ContainerCommandResponseProto.Builder builder =
+ ContainerUtils.getSuccessResponseBuilder(msg);
+ builder.setGetBlock(getBlock);
+ return builder.build();
+ }
+
+ /**
+ * Returns successful getCommittedBlockLength Response.
+ * @param msg - Request.
+ * @return Response.
+ */
+ public static ContainerCommandResponseProto getBlockLengthResponse(
+ ContainerCommandRequestProto msg, long blockLength) {
+ GetCommittedBlockLengthResponseProto.Builder
+ committedBlockLengthResponseBuilder =
+ getCommittedBlockLengthResponseBuilder(blockLength,
+ msg.getGetCommittedBlockLength().getBlockID());
+ ContainerProtos.ContainerCommandResponseProto.Builder builder =
+ ContainerUtils.getSuccessResponseBuilder(msg);
+ builder.setGetCommittedBlockLength(committedBlockLengthResponseBuilder);
+ return builder.build();
+ }
+
+ private static GetCommittedBlockLengthResponseProto.Builder
+ getCommittedBlockLengthResponseBuilder(long blockLength,
+ ContainerProtos.DatanodeBlockID blockID) {
+ ContainerProtos.GetCommittedBlockLengthResponseProto.Builder
+ getCommittedBlockLengthResponseBuilder = ContainerProtos.
+ GetCommittedBlockLengthResponseProto.newBuilder();
+ getCommittedBlockLengthResponseBuilder.setBlockLength(blockLength);
+ getCommittedBlockLengthResponseBuilder.setBlockID(blockID);
+ return getCommittedBlockLengthResponseBuilder;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java
deleted file mode 100644
index a83d298..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyUtils.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.keyvalue.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .GetKeyResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- GetCommittedBlockLengthResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.
- PutKeyResponseProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.utils.MetadataStore;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.NO_SUCH_KEY;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Result.UNABLE_TO_READ_METADATA_DB;
-
-/**
- * Utils functions to help key functions.
- */
-public final class KeyUtils {
-
- /** Never constructed. **/
- private KeyUtils() {
-
- }
- /**
- * Get a DB handler for a given container.
- * If the handler doesn't exist in cache yet, first create one and
- * add into cache. This function is called with containerManager
- * ReadLock held.
- *
- * @param containerData containerData.
- * @param conf configuration.
- * @return MetadataStore handle.
- * @throws StorageContainerException
- */
- public static MetadataStore getDB(KeyValueContainerData containerData,
- Configuration conf) throws
- StorageContainerException {
- Preconditions.checkNotNull(containerData);
- ContainerCache cache = ContainerCache.getInstance(conf);
- Preconditions.checkNotNull(cache);
- Preconditions.checkNotNull(containerData.getDbFile());
- try {
- return cache.getDB(containerData.getContainerID(), containerData
- .getContainerDBType(), containerData.getDbFile().getAbsolutePath());
- } catch (IOException ex) {
- String message = String.format("Error opening DB. Container:%s " +
- "ContainerPath:%s", containerData.getContainerID(), containerData
- .getDbFile().getPath());
- throw new StorageContainerException(message, UNABLE_TO_READ_METADATA_DB);
- }
- }
- /**
- * Remove a DB handler from cache.
- *
- * @param container - Container data.
- * @param conf - Configuration.
- */
- public static void removeDB(KeyValueContainerData container, Configuration
- conf) {
- Preconditions.checkNotNull(container);
- ContainerCache cache = ContainerCache.getInstance(conf);
- Preconditions.checkNotNull(cache);
- cache.removeDB(container.getContainerID());
- }
-
- /**
- * Shutdown all DB Handles.
- *
- * @param cache - Cache for DB Handles.
- */
- @SuppressWarnings("unchecked")
- public static void shutdownCache(ContainerCache cache) {
- cache.shutdownCache();
- }
-
- /**
- * Parses the {@link KeyData} from a bytes array.
- *
- * @param bytes key data in bytes.
- * @return key data.
- * @throws IOException if the bytes array is malformed or invalid.
- */
- public static KeyData getKeyData(byte[] bytes) throws IOException {
- try {
- ContainerProtos.KeyData keyData = ContainerProtos.KeyData.parseFrom(
- bytes);
- KeyData data = KeyData.getFromProtoBuf(keyData);
- return data;
- } catch (IOException e) {
- throw new StorageContainerException("Failed to parse key data from the" +
- " bytes array.", NO_SUCH_KEY);
- }
- }
-
- /**
- * Returns putKey response success.
- * @param msg - Request.
- * @return Response.
- */
- public static ContainerCommandResponseProto putKeyResponseSuccess(
- ContainerCommandRequestProto msg, long blockLength) {
- GetCommittedBlockLengthResponseProto.Builder
- committedBlockLengthResponseBuilder =
- getCommittedBlockLengthResponseBuilder(blockLength,
- msg.getPutKey().getKeyData().getBlockID());
- PutKeyResponseProto.Builder putKeyResponse =
- PutKeyResponseProto.newBuilder();
- putKeyResponse
- .setCommittedBlockLength(committedBlockLengthResponseBuilder);
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getSuccessResponseBuilder(msg);
- builder.setPutKey(putKeyResponse);
- return builder.build();
- }
- /**
- * Returns successful keyResponse.
- * @param msg - Request.
- * @return Response.
- */
- public static ContainerCommandResponseProto getKeyResponseSuccess(
- ContainerCommandRequestProto msg) {
- return ContainerUtils.getSuccessResponse(msg);
- }
-
-
- public static ContainerCommandResponseProto getKeyDataResponse(
- ContainerCommandRequestProto msg, KeyData data) {
- GetKeyResponseProto.Builder getKey = ContainerProtos
- .GetKeyResponseProto
- .newBuilder();
- getKey.setKeyData(data.getProtoBufMessage());
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getSuccessResponseBuilder(msg);
- builder.setGetKey(getKey);
- return builder.build();
- }
-
- /**
- * Returns successful getCommittedBlockLength Response.
- * @param msg - Request.
- * @return Response.
- */
- public static ContainerCommandResponseProto getBlockLengthResponse(
- ContainerCommandRequestProto msg, long blockLength) {
- GetCommittedBlockLengthResponseProto.Builder
- committedBlockLengthResponseBuilder =
- getCommittedBlockLengthResponseBuilder(blockLength,
- msg.getGetCommittedBlockLength().getBlockID());
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getSuccessResponseBuilder(msg);
- builder.setGetCommittedBlockLength(committedBlockLengthResponseBuilder);
- return builder.build();
- }
-
- private static GetCommittedBlockLengthResponseProto.Builder
- getCommittedBlockLengthResponseBuilder(
- long blockLength, ContainerProtos.DatanodeBlockID blockID) {
- ContainerProtos.GetCommittedBlockLengthResponseProto.Builder
- getCommittedBlockLengthResponseBuilder = ContainerProtos.
- GetCommittedBlockLengthResponseProto.newBuilder();
- getCommittedBlockLengthResponseBuilder.setBlockLength(blockLength);
- getCommittedBlockLengthResponseBuilder.setBlockID(blockID);
- return getCommittedBlockLengthResponseBuilder;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index ed71ea2..0243443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends
private Container masterContainer = null;
private ByteBuffer appAttemptTokens = null;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationAttemptStateDataPBImpl() {
builder = ApplicationAttemptStateDataProto.newBuilder();
@@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public long getMemorySeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getMemorySeconds();
+ return getGuaranteedMemorySeconds();
}
@Override
+ @Deprecated
public long getVcoreSeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getVcoreSeconds();
+ return getGuaranteedVcoreSeconds();
}
@Override
+ @Deprecated
public void setMemorySeconds(long memorySeconds) {
- maybeInitBuilder();
- builder.setMemorySeconds(memorySeconds);
+ setGuaranteedMemorySeconds(memorySeconds);
}
@Override
+ @Deprecated
public void setVcoreSeconds(long vcoreSeconds) {
+ setGuaranteedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
+ public long getGuaranteedMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public void setGuaranteedMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public long getGuaranteedVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getVcoreSeconds();
+ }
+
+ @Override
+ public void setGuaranteedVcoreSeconds(long vcoreSeconds) {
maybeInitBuilder();
builder.setVcoreSeconds(vcoreSeconds);
}
@Override
+ public long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
public long getPreemptedMemorySeconds() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getPreemptedMemorySeconds();
@@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ @Deprecated
+ public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ setGuaranteedResourceSecondsMap(resourceSecondsMap);
+ }
+
+ @Override
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
- p.getApplicationResourceUsageMapList());
- return this.resourceSecondsMap;
+ this.guaranteedResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationResourceUsageMapList());
+ return this.guaranteedResourceSecondsMap;
}
@Override
- public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ public void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
maybeInitBuilder();
builder.clearApplicationResourceUsageMap();
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap != null) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
@@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
}
+
+ @Override
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
+ maybeInitBuilder();
+ builder.clearApplicationOpportunisticResourceUsageMap();
+ this.opportunisticResourceSecondsMap = resourceSecondsMap;
+ if (resourceSecondsMap != null) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 42e2bcf..7eb2379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -456,6 +456,7 @@ public class RMAppImpl implements RMApp, Recoverable {
} else {
this.startTime = startTime;
}
+ LOG.info(applicationId.toString() + " started at " + this.startTime);
this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
@@ -775,10 +776,12 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppMetrics rmAppMetrics = getRMAppMetrics();
- appUsageReport
- .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
+ appUsageReport.setGuaranteedResourceSecondsMap(
+ rmAppMetrics.getGuaranteedResourceSecondsMap());
appUsageReport.setPreemptedResourceSecondsMap(
rmAppMetrics.getPreemptedResourceSecondsMap());
+ appUsageReport.setOpportunisticResourceSecondsMap(
+ rmAppMetrics.getOpportunisticResourceSecondsMap());
}
if (currentApplicationAttemptId == null) {
@@ -1705,8 +1708,9 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2);
+ Map<String, Long> preemptedSecondsMap = new HashMap<>(2);
+ Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2);
this.readLock.lock();
try {
for (RMAppAttempt attempt : attempts.values()) {
@@ -1722,20 +1726,15 @@ public class RMAppImpl implements RMApp, Recoverable {
// for both running and finished containers.
AggregateAppResourceUsage resUsage =
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
- for (Map.Entry<String, Long> entry : resUsage
- .getResourceUsageSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- resourceSecondsMap.put(entry.getKey(), value);
- }
- for (Map.Entry<String, Long> entry : attemptMetrics
- .getPreemptedResourceSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- preemptedSecondsMap.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ guaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap(),
+ opportunsiticResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ attemptMetrics.getPreemptedResourceSecondsMap(),
+ preemptedSecondsMap);
}
}
} finally {
@@ -1743,7 +1742,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
- numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
+ numAMContainerPreempted, guaranteedResourceSecondsMap,
+ preemptedSecondsMap, opportunsiticResourceSecondsMap);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 2bb7fd1..76272e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -28,18 +28,21 @@ public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
- private final Map<String, Long> resourceSecondsMap;
+ private final Map<String, Long> guaranteedResourceSecondsMap;
private final Map<String, Long> preemptedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
}
public Resource getResourcePreempted() {
@@ -54,17 +57,25 @@ public class RMAppMetrics {
return numAMContainersPreempted;
}
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
}
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
public long getPreemptedMemorySeconds() {
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
@@ -75,12 +86,15 @@ public class RMAppMetrics {
ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceSecondsMap() {
- return resourceSecondsMap;
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ return guaranteedResourceSecondsMap;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
index b858712..a75cc57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -27,30 +27,55 @@ import java.util.Map;
@Private
public class AggregateAppResourceUsage {
- private Map<String, Long> resourceSecondsMap = new HashMap<>();
+ private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
- public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap.putAll(resourceSecondsMap);
+ public AggregateAppResourceUsage(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap.putAll(
+ guaranteedResourceSecondsMap);
+ this.opportunisticResourceSecondsMap.putAll(
+ opportunisticResourceSecondsMap);
}
/**
- * @return the memorySeconds
+ * @return the guaranteed memory seconds
*/
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
/**
- * @return the vcoreSeconds
+ * @return the guaranteed vcore seconds
*/
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceUsageSecondsMap() {
- return resourceSecondsMap;
+ /**
+ * @return the opportunistic memory seconds
+ */
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+
+ /**
+ * @return the opportunistic vcore seconds
+ */
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 3ec9c49..b0fa02f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -938,15 +938,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock.lock();
try {
ApplicationResourceUsageReport report =
- scheduler.getAppResourceUsageReport(this.getAppAttemptId());
+ scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId());
if (report == null) {
report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
}
AggregateAppResourceUsage resUsage =
this.attemptMetrics.getAggregateAppResourceUsage();
- report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
+ report.setGuaranteedResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap());
report.setPreemptedResourceSecondsMap(
this.attemptMetrics.getPreemptedResourceSecondsMap());
+ report.setOpportunisticResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap());
return report;
} finally {
this.readLock.unlock();
@@ -983,10 +986,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
- this.attemptMetrics
- .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage(
+ attemptState.getGuaranteedResourceSecondsMap());
this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
attemptState.getPreemptedResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage(
+ attemptState.getOpportunisticResourceSecondsMap());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1406,9 +1411,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
- stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
- getFinishTime(), resUsage.getResourceUsageSecondsMap(),
- this.attemptMetrics.getPreemptedResourceSecondsMap());
+ stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus,
+ exitStatus, getFinishTime(),
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ this.attemptMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
@@ -1837,8 +1844,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
newTrackingUrl, appAttempt.getDiagnostics(), null,
ContainerExitStatus.INVALID, appAttempt.getFinishTime(),
appAttempt.attemptMetrics.getAggregateAppResourceUsage()
- .getResourceUsageSecondsMap(),
- appAttempt.attemptMetrics.getPreemptedResourceSecondsMap());
+ .getGuaranteedResourceUsageSecondsMap(),
+ appAttempt.attemptMetrics.getPreemptedResourceSecondsMap(),
+ appAttempt.attemptMetrics.getAggregateAppResourceUsage()
+ .getOpportunisticResourceSecondsMap());
appAttempt.rmContext.getStateStore()
.updateApplicationAttemptState(attemptState);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 43e6e4d..59fed88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -54,8 +54,11 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
- private Map<String, AtomicLong> resourceUsageMap = new ConcurrentHashMap<>();
- private Map<String, AtomicLong> preemptedResourceMap = new ConcurrentHashMap<>();
+ private Map<String, AtomicLong> guaranteedResourceUsageMap
+ = new HashMap<>();
+ private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
+ private Map<String, AtomicLong> opportunisticResourceUsageMap
+ = new HashMap<>();
private RMContext rmContext;
private int[][] localityStatistics =
@@ -130,8 +133,10 @@ public class RMAppAttemptMetrics {
}
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
- Map<String, Long> resourcesUsed =
- convertAtomicLongMaptoLongMap(resourceUsageMap);
+ Map<String, Long> guaranteedResourcesUsed =
+ convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap);
+ Map<String, Long> opportunisticResourcesUsed =
+ convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap);
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
@@ -140,27 +145,25 @@ public class RMAppAttemptMetrics {
if (currentAttempt != null
&& currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport =
- rmContext.getScheduler().getAppResourceUsageReport(attemptId);
+ rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId);
if (appResUsageReport != null) {
- Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
- for (Map.Entry<String, Long> entry : tmp.entrySet()) {
- Long value = resourcesUsed.get(entry.getKey());
- if (value != null) {
- value += entry.getValue();
- } else {
- value = entry.getValue();
- }
- resourcesUsed.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getGuaranteedResourceSecondsMap(),
+ guaranteedResourcesUsed);
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getOpportunisticResourceSecondsMap(),
+ opportunisticResourcesUsed);
}
}
}
- return new AggregateAppResourceUsage(resourcesUsed);
+
+ return new AggregateAppResourceUsage(guaranteedResourcesUsed,
+ opportunisticResourcesUsed);
}
public void updateAggregateAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
- updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
+ updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
@@ -168,9 +171,14 @@ public class RMAppAttemptMetrics {
updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
}
- public void updateAggregateAppResourceUsage(
+ public void updateAggregateAppGuaranteedResourceUsage(
Map<String, Long> resourceSecondsMap) {
- updateUsageMap(resourceSecondsMap, resourceUsageMap);
+ updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap);
+ }
+
+ public void updateAggregateAppOpportunisticResourceUsage(
+ Map<String, Long> resourceSecondsMap) {
+ updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(
@@ -200,10 +208,9 @@ public class RMAppAttemptMetrics {
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
-
}
resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.set(entry.getValue());
+ resourceUsed.addAndGet(entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
new file mode 100644
index 0000000..c91dae8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import java.util.Map;
+
+/**
+ * The resource usage information of a <code>Container</code>.
+ */
+@Private
+public class ContainerResourceUsageReport {
+ private final Map<String, Long> guaranteedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
+
+ public ContainerResourceUsageReport(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index 8f751b0..4897312 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -87,7 +87,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>,
ContainerState getContainerState();
ContainerReport createContainerReport();
-
+
+ ContainerResourceUsageReport getResourceUsageReport();
+
boolean isAMContainer();
ContainerRequest getContainerRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 945e7cb..ccd02ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -39,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -176,7 +179,7 @@ public class RMContainerImpl implements RMContainer {
private NodeId reservedNode;
private SchedulerRequestKey reservedSchedulerKey;
private long creationTime;
- private long finishTime;
+ private long finishTime = -1L;
private ContainerStatus finishedStatus;
private boolean isAMContainer;
private ContainerRequest containerRequestForRecovery;
@@ -191,6 +194,10 @@ public class RMContainerImpl implements RMContainer {
private volatile Set<String> allocationTags = null;
+ // This is updated whenever the container instance is updated in
+ // the case of container increase/decrease/promotion/demotion
+ private long lastContainerUpdateTime;
+
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
@@ -232,6 +239,7 @@ public class RMContainerImpl implements RMContainer {
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
+ this.lastContainerUpdateTime = creationTime;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
@@ -272,9 +280,87 @@ public class RMContainerImpl implements RMContainer {
}
public void setContainer(Container container) {
+ // containers are updated by resetting the underlying Container instance,
+ // always update RMContainer resource usage before the update.
+ updateAppAttemptMetrics(true);
this.container = container;
}
+ /**
+ * Collect the resource usage information of the current underlying Container
+ * instance and update its RMAppAttemptMetrics. This is called whenever the
+ * RMContainer is updated (by updating its underlying container) or finishes.
+ * @param containerUpdated if it is called upon container update event
+ */
+ private void updateAppAttemptMetrics(boolean containerUpdated) {
+ RMAppAttempt rmAttempt = rmContext.getRMApps()
+ .get(getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
+
+ if (rmAttempt != null) {
+ // collect resource usage information of the current Container instance
+ ContainerResourceUsageReport resourceUsage =
+ getCurrentContainerResourceUsage(containerUpdated);
+
+ // If this is a preempted container, update preemption metrics
+ if (finishedStatus != null &&
+ ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics()
+ .updatePreemptionInfo(container.getResource(), this);
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getOpportunisticResourceSecondsMap());
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppGuaranteedResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ }
+
+ /**
+ * Get resource usage of the current underlying Container instance.
+ * @param containerUpdated if this is called upon container update
+ * @return resource usage of the current <code>container</code>
+ */
+ private ContainerResourceUsageReport getCurrentContainerResourceUsage(
+ boolean containerUpdated) {
+ ContainerResourceUsageReport report;
+
+ // A container generates usage until it finishes which is indicated
+ // by a positive finish timestamp that is set once it finishes
+ final long currentTimeMillis =
+ finishTime < 0 ? System.currentTimeMillis() : finishTime;
+ final long usedSeconds = (currentTimeMillis -
+ lastContainerUpdateTime) / 1000;
+ Resource resource = container.getResource();
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ guaranteedResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(guaranteedResourceSeconds,
+ Collections.emptyMap());
+ } else {
+ Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ opportunisticResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(Collections.emptyMap(),
+ opportunisticResourceSeconds);
+ }
+
+ if (containerUpdated) {
+ lastContainerUpdateTime = currentTimeMillis;
+ }
+ return report;
+ }
+
@Override
public RMContainerState getState() {
this.readLock.lock();
@@ -704,7 +790,7 @@ public class RMContainerImpl implements RMContainer {
// Inform AppAttempt
// container.getContainer() can return null when a RMContainer is a
// reserved container
- updateAttemptMetrics(container);
+ container.updateAppAttemptMetrics(false);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
@@ -721,28 +807,6 @@ public class RMContainerImpl implements RMContainer {
container, container.finishTime);
}
}
-
- private static void updateAttemptMetrics(RMContainerImpl container) {
- Resource resource = container.getContainer().getResource();
- RMApp app = container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId());
- if (app != null) {
- RMAppAttempt rmAttempt = app.getCurrentAppAttempt();
- if (rmAttempt != null) {
- long usedMillis = container.finishTime - container.creationTime;
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregateAppResourceUsage(resource, usedMillis);
- // If this is a preempted container, update preemption metrics
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics()
- .updatePreemptionInfo(resource, container);
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
- }
- }
- }
- }
}
private static boolean shouldPublishNonAMContainerEventstoATS(
@@ -804,6 +868,11 @@ public class RMContainerImpl implements RMContainer {
}
@Override
+ public ContainerResourceUsageReport getResourceUsageReport() {
+ return getCurrentContainerResourceUsage(false);
+ }
+
+ @Override
public String getNodeHttpAddress() {
try {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 278e474..a8f5093 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -417,7 +417,7 @@ public abstract class AbstractYarnScheduler
}
@Override
- public ApplicationResourceUsageReport getAppResourceUsageReport(
+ public ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId) {
SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
if (attempt == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f9df2b8..25a9415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -61,12 +61,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -108,7 +108,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
- private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> lastGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ private Map<String, Long> lastOpportunisticResourceSecondsMap =
+ new HashMap<>();
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
@@ -1083,34 +1086,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// recently.
if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
> MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> aggregatedGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ Map<String, Long> aggregatedOpportunisticResourceSecondsMap =
+ new HashMap<>();
for (RMContainer rmContainer : this.liveContainers.values()) {
- long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
- Resource resource = rmContainer.getContainer().getResource();
- for (ResourceInformation entry : resource.getResources()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
- value += entry.getValue() * usedMillis
- / DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getName(), value);
- }
+ ContainerResourceUsageReport containerResourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ aggregatedGuaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getOpportunisticResourceSecondsMap(),
+ aggregatedOpportunisticResourceSecondsMap);
}
-
+ lastGuaranteedResourceSecondsMap =
+ aggregatedGuaranteedResourceSecondsMap;
+ lastOpportunisticResourceSecondsMap =
+ aggregatedOpportunisticResourceSecondsMap;
lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
- lastResourceSecondsMap = resourceSecondsMap;
}
- return new AggregateAppResourceUsage(lastResourceSecondsMap);
+ return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap,
+ lastOpportunisticResourceSecondsMap);
}
+ /**
+ * Get the resources that are actively being used by the app attempt.
+ * @return active resource usage
+ */
public ApplicationResourceUsageReport getResourceUsageReport() {
try {
writeLock.lock();
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
- Resource usedResourceClone = Resources.clone(
+ Resource guaranteedResourceUsedClone = Resources.clone(
attemptResourceUsage.getAllUsed());
+ Resource opportunisticResourceUsedClone = Resources.clone(
+ attemptOpportunisticResourceUsage.getAllUsed());
Resource reservedResourceClone = Resources.clone(
attemptResourceUsage.getReserved());
+ Resource neededResource = Resources.add(opportunisticResourceUsedClone,
+ Resources.add(guaranteedResourceUsedClone, reservedResourceClone));
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
@@ -1125,18 +1141,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
float queueCapacityPerc = queue.getQueueInfo(false, false)
.getCapacity();
if (queueCapacityPerc != 0) {
- queueUsagePerc = calc.divide(cluster, usedResourceClone,
+ queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone,
Resources.multiply(cluster, queueCapacityPerc)) * 100;
}
clusterUsagePerc =
- calc.divide(cluster, usedResourceClone, cluster) * 100;
+ calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100;
}
return ApplicationResourceUsageReport
.newInstance(liveContainers.size(), reservedContainers.size(),
- usedResourceClone, reservedResourceClone,
- Resources.add(usedResourceClone, reservedResourceClone),
- runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
- clusterUsagePerc, preemptedResourceSecondsMaps);
+ guaranteedResourceUsedClone, reservedResourceClone,
+ neededResource,
+ runningResourceUsage.getGuaranteedResourceUsageSecondsMap(),
+ queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps,
+ opportunisticResourceUsedClone,
+ runningResourceUsage.getOpportunisticResourceSecondsMap());
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 0f7a5b5..2a52389 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -172,7 +172,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
*/
@LimitedPrivate("yarn")
@Evolving
- ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index e6a71f6..4bd22584 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1111,7 +1111,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
.getEffectiveCapacity(getAppAMNodePartitionName());
if (!effCap.equals(Resources.none())) {
queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(), effCap) * 100;
+ report.getGuaranteedResourcesUsed(), effCap) * 100;
}
report.setQueueUsagePercentage(queueUsagePerc);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0effd42..7b32a9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
queue.getMetrics().releaseResources(
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
- this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedGuaranteedResource(containerResource);
+ if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
+ } else {
+ this.attemptOpportunisticResourceUsage.decUsed(containerResource);
+ }
+
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 14ad277..3a5c41e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -96,6 +96,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
th(".runningcontainer", "Running Containers").
th(".allocatedCpu", "Allocated CPU VCores").
th(".allocatedMemory", "Allocated Memory MB").
+ th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU Vcores").
+ th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB").
th(".reservedCpu", "Reserved CPU VCores").
th(".reservedMemory", "Reserved Memory MB").
th(".progress", "Progress").
@@ -146,6 +150,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.valueOf(appInfo.getAllocatedVCores())).append("\",\"")
.append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
.valueOf(appInfo.getAllocatedMB())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"")
.append(appInfo.getReservedVCores() == -1 ? "N/A" : String
.valueOf(appInfo.getReservedVCores())).append("\",\"")
.append(appInfo.getReservedMB() == -1 ? "N/A" : String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index d260400..d5e8d3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
.__("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
- .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
- StringHelper
- .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
+ .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getGuaranteedResourceSecondsMap()))
+ .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getOpportunisticResourceSecondsMap()))
.__("Aggregate Preempted Resource Allocation:",
appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
appMetrics.getPreemptedResourceSecondsMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index b1c0cd9..b937c41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -70,6 +70,10 @@ public class RMAppsBlock extends AppsBlock {
.th(".allocatedMemory", "Allocated Memory MB")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
+ .th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU VCores")
+ .th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
@@ -157,6 +161,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getReservedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getReservedMemoryMB()))
.append("\",\"")
+ .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedOpportunisticCpuVcores()))
+ .append("\",\"")
+ .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" :
+ String.valueOf(app.getAllocatedOpportunisticMemoryMB()))
+ .append("\",\"")
.append(queuePercent)
.append("\",\"")
.append(clusterPercent)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 63b6fe0..14795d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -97,6 +97,8 @@ public class AppInfo {
private String masterNodeId;
private long allocatedMB;
private long allocatedVCores;
+ private long allocatedOpportunisticMB;
+ private long allocatedOpportunisticVCores;
private long reservedMB;
private long reservedVCores;
private int runningContainers;
@@ -105,6 +107,9 @@ public class AppInfo {
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
protected Map<String, Long> resourceSecondsMap;
+ private long opportunisticMemorySeconds;
+ private long opportunisticVcoreSeconds;
+ private Map<String, Long> opportunisticResourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@@ -205,10 +210,17 @@ public class AppInfo {
ApplicationResourceUsageReport resourceReport =
attempt.getApplicationResourceUsageReport();
if (resourceReport != null) {
- Resource usedResources = resourceReport.getUsedResources();
+ Resource guaranteedResourcesUsed =
+ resourceReport.getGuaranteedResourcesUsed();
+ Resource opportunisticResourceUsed =
+ resourceReport.getOpportunisticResourcesUsed();
Resource reservedResources = resourceReport.getReservedResources();
- allocatedMB = usedResources.getMemorySize();
- allocatedVCores = usedResources.getVirtualCores();
+ allocatedMB = guaranteedResourcesUsed.getMemorySize();
+ allocatedVCores = guaranteedResourcesUsed.getVirtualCores();
+ allocatedOpportunisticMB =
+ opportunisticResourceUsed.getMemorySize();
+ allocatedOpportunisticVCores =
+ opportunisticResourceUsed.getVirtualCores();
reservedMB = reservedResources.getMemorySize();
reservedVCores = reservedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
@@ -251,9 +263,13 @@ public class AppInfo {
numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
- memorySeconds = appMetrics.getMemorySeconds();
- vcoreSeconds = appMetrics.getVcoreSeconds();
- resourceSecondsMap = appMetrics.getResourceSecondsMap();
+ memorySeconds = appMetrics.getGuaranteedMemorySeconds();
+ vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds();
+ resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap();
+ opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds();
+ opportunisticResourceSecondsMap =
+ appMetrics.getOpportunisticResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
@@ -470,6 +486,13 @@ public class AppInfo {
return this.allocatedVCores;
}
+ public long getAllocatedOpportunisticMB() {
+ return this.allocatedOpportunisticMB;
+ }
+
+ public long getAllocatedOpportunisticVCores() {
+ return this.allocatedOpportunisticVCores;
+ }
public long getReservedMB() {
return this.reservedMB;
}
@@ -494,18 +517,30 @@ public class AppInfo {
return numAMContainerPreempted;
}
- public long getMemorySeconds() {
+ public long getGuaranteedMemorySeconds() {
return memorySeconds;
}
- public long getVcoreSeconds() {
+ public long getGuaranteedVcoreSeconds() {
return vcoreSeconds;
}
- public Map<String, Long> getResourceSecondsMap() {
+ public long getOpportunisticMemorySeconds() {
+ return opportunisticMemorySeconds;
+ }
+
+ public long getOpportunisticVcoreSeconds() {
+ return opportunisticVcoreSeconds;
+ }
+
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
return resourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@@ -598,6 +633,15 @@ public class AppInfo {
this.allocatedVCores = allocatedVCores;
}
+ public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) {
+ this.allocatedOpportunisticMB = allocatedOpportunisticMB;
+ }
+
+ public void setAllocatedOpportunisticVCores(
+ long allocatedOpportunisticVCores) {
+ this.allocatedOpportunisticVCores = allocatedOpportunisticVCores;
+ }
+
public void setReservedMB(long reservedMB) {
this.reservedMB = reservedMB;
}
@@ -610,12 +654,20 @@ public class AppInfo {
this.runningContainers = runningContainers;
}
- public void setMemorySeconds(long memorySeconds) {
- this.memorySeconds = memorySeconds;
+ public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) {
+ this.memorySeconds = guaranteedMemorySeconds;
+ }
+
+ public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) {
+ this.vcoreSeconds = guaranteedVcoreSeconds;
+ }
+
+ public void setOpportunisticMemorySeconds(long oppMemorySeconds) {
+ this.opportunisticMemorySeconds = oppMemorySeconds;
}
- public void setVcoreSeconds(long vcoreSeconds) {
- this.vcoreSeconds = vcoreSeconds;
+ public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) {
+ this.opportunisticVcoreSeconds = oppVcoreSeconds;
}
public void setAppId(String appId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 35c77ab..3763339 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -90,6 +90,7 @@ message ApplicationAttemptStateDataProto {
optional int64 preempted_vcore_seconds = 14;
repeated StringLongMapProto application_resource_usage_map = 15;
repeated StringLongMapProto preempted_resource_usage_map = 16;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 17;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8617dc8..febea7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -1179,12 +1179,20 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
- 10, 1, resourceSecondsMap, new HashMap<>());
+ 10, 1, guaranteedResourceSecondsMap,
+ new HashMap<>(), opportunisticResourceSecondsMap);
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
@@ -1195,16 +1203,19 @@ public class TestAppManager{
Assert.assertFalse(msg.contains("\r"));
String escaped = "\\n\\n\\r\\r";
- assertTrue(msg.contains("Multiline" + escaped +"AppName"));
- assertTrue(msg.contains("Multiline" + escaped +"UserName"));
- assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
- assertTrue(msg.contains("submitTime=1000"));
- assertTrue(msg.contains("memorySeconds=16384"));
- assertTrue(msg.contains("vcoreSeconds=64"));
- assertTrue(msg.contains("preemptedAMContainers=1"));
- assertTrue(msg.contains("preemptedNonAMContainers=10"));
- assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
- assertTrue(msg.contains("applicationType=MAPREDUCE"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+ Assert.assertTrue(msg.contains("submitTime=1000"));
+ Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+ Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+ Assert.assertTrue(
+ msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+ Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c..698ca90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
- -1, usageReport.getUsedResources().getMemorySize());
+ -1, usageReport.getGuaranteedResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemorySize());
+ Assert.assertEquals("Enemy should not see app used opportunistic resources",
+ -1, usageReport.getOpportunisticResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 61048a5..a2ca4b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -385,8 +385,8 @@ public class TestClientRMService {
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
- Assert.assertEquals(10, usageReport.getMemorySeconds());
- Assert.assertEquals(3, usageReport.getVcoreSeconds());
+ Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds());
Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
@@ -1421,26 +1421,28 @@ public class TestClientRMService {
when(asContext.getMaxAppAttempts()).thenReturn(1);
when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
- RMAppImpl app =
- spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
+ ResourceRequest resourceRequest = BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ Resource.newInstance(1024, 1), 1);
+ RMAppImpl rmApp =
+ new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- Collections.singletonList(BuilderUtils.newResourceRequest(
- RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1))){
- @Override
- public ApplicationReport createAndGetApplicationReport(
- String clientUserName, boolean allowAccess) {
- ApplicationReport report = super.createAndGetApplicationReport(
- clientUserName, allowAccess);
- ApplicationResourceUsageReport usageReport =
- report.getApplicationResourceUsageReport();
- usageReport.setMemorySeconds(memorySeconds);
- usageReport.setVcoreSeconds(vcoreSeconds);
- report.setApplicationResourceUsageReport(usageReport);
- return report;
- }
- });
+ Collections.singletonList(resourceRequest)) {
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationReport report = super.createAndGetApplicationReport(
+ clientUserName, allowAccess);
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ usageReport.setGuaranteedMemorySeconds(memorySeconds);
+ usageReport.setGuaranteedVcoreSeconds(vcoreSeconds);
+ report.setApplicationResourceUsageReport(usageReport);
+ return report;
+ }
+ };
+ RMAppImpl app = spy(rmApp);
app.getAMResourceRequests().get(0)
.setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
@@ -1496,7 +1498,8 @@ public class TestClientRMService {
when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+ when(yarnScheduler.getAppActiveResourceUsageReport(attemptId))
+ .thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-8240. Add queue-level control
to allow all applications in a queue to opt-out. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-8240. Add queue-level control to allow all applications in a queue to opt-out. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23ae827c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23ae827c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23ae827c
Branch: refs/heads/YARN-1011
Commit: 23ae827cff9521025dc267ed7e0c3ccb3d090ed7
Parents: 4cf8e47
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Jun 4 11:20:32 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:28:58 2018 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 9 ++
.../scheduler/fair/FSLeafQueue.java | 5 +
.../resourcemanager/scheduler/fair/FSQueue.java | 21 +++-
.../allocation/AllocationFileQueueParser.java | 4 +
.../fair/allocation/QueueProperties.java | 13 +++
.../webapp/dao/FairSchedulerQueueInfo.java | 6 ++
.../fair/TestAllocationFileLoaderService.java | 61 +++++++++++
.../scheduler/fair/TestFairScheduler.java | 103 +++++++++++++++++++
.../scheduler/fair/TestQueueManager.java | 70 +++++++++++++
.../src/site/markdown/FairScheduler.md | 1 +
10 files changed, 292 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index e48e04b..14e2aeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -103,6 +104,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
private ReservationQueueConfiguration globalReservationQueueConfig;
private final Set<String> nonPreemptableQueues;
+ private final Map<String, Boolean> queueOversubscriptionSetting;
public AllocationConfiguration(QueueProperties queueProperties,
AllocationFileParser allocationFileParser,
@@ -138,6 +140,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
this.placementPolicy = newPlacementPolicy;
this.configuredQueues = queueProperties.getConfiguredQueues();
this.nonPreemptableQueues = queueProperties.getNonPreemptableQueues();
+ this.queueOversubscriptionSetting =
+ queueProperties.getQueueOversubscriptionSetting();
}
public AllocationConfiguration(Configuration conf) {
@@ -167,6 +171,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
placementPolicy =
QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
nonPreemptableQueues = new HashSet<>();
+ queueOversubscriptionSetting = new HashMap<>(0);
}
/**
@@ -221,6 +226,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return !nonPreemptableQueues.contains(queueName);
}
+ public Optional<Boolean> isOversubscriptionAllowed(String queueName) {
+ return Optional.ofNullable(queueOversubscriptionSetting.get(queueName));
+ }
+
private float getQueueWeight(String queue) {
Float weight = queueWeights.get(queue);
return (weight == null) ? 1.0f : weight;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 3834130..48836b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -378,6 +378,11 @@ public class FSLeafQueue extends FSQueue {
getName() + " fairShare: " + getFairShare());
}
+ if (opportunistic && !isOversubscriptionAllowed()) {
+ // do not proceed if the queue is opted out of over-subscription
+ return assigned;
+ }
+
if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 52c76f4..3a9ffae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -75,6 +76,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private ConfigurableResource maxShare;
protected int maxRunningApps;
private ConfigurableResource maxChildQueueResource;
+ private boolean isOversubscriptionAllowed;
// maxAMShare is a value between 0 and 1.
protected float maxAMShare;
@@ -93,6 +95,7 @@ public abstract class FSQueue implements Queue, Schedulable {
this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, name);
this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
this.parent = parent;
+ this.isOversubscriptionAllowed = true;
setPolicy(scheduler.getAllocationConfiguration().getSchedulingPolicy(name));
reinit(false);
}
@@ -112,6 +115,14 @@ public abstract class FSQueue implements Queue, Schedulable {
allocConf.initFSQueue(this);
updatePreemptionVariables();
+ Optional<Boolean> allowed = allocConf.isOversubscriptionAllowed(name);
+ if (allowed.isPresent()) {
+ isOversubscriptionAllowed = allowed.get();
+ } else if (getParent() != null){
+ // inherit from the parent queue if it is not explicitly set
+ isOversubscriptionAllowed = getParent().isOversubscriptionAllowed();
+ }
+
if (recursive) {
for (FSQueue child : getChildQueues()) {
child.reinit(recursive);
@@ -213,6 +224,15 @@ public abstract class FSQueue implements Queue, Schedulable {
return 0;
}
+ public boolean isOversubscriptionAllowed() {
+ return isOversubscriptionAllowed;
+ }
+
+ @VisibleForTesting
+ public void allowOversubscription(boolean allowed) {
+ this.isOversubscriptionAllowed = allowed;
+ }
+
@Override
public Priority getPriority() {
Priority p = recordFactory.newRecordInstance(Priority.class);
@@ -584,7 +604,6 @@ public abstract class FSQueue implements Queue, Schedulable {
return sb.toString();
}
-
/**
* Recursively dump states of all queues.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
index 441c34a..62a6060 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
@@ -68,6 +68,7 @@ public class AllocationFileQueueParser {
private static final String ACL_SUBMIT_RESERVATIONS = "aclSubmitReservations";
private static final String RESERVATION = "reservation";
private static final String ALLOW_PREEMPTION_FROM = "allowPreemptionFrom";
+ private static final String ALLOW_OVERSUBSCRIPTION ="allowOversubscription";
private static final String QUEUE = "queue";
private static final String POOL = "pool";
@@ -207,6 +208,9 @@ public class AllocationFileQueueParser {
if (!Boolean.parseBoolean(text)) {
builder.nonPreemptableQueues(queueName);
}
+ } else if (ALLOW_OVERSUBSCRIPTION.equals(field.getTagName())) {
+ Boolean allow = Boolean.parseBoolean(getTrimmedTextData(field));
+ builder.addQueueOversubscriptionSetting(queueName, allow);
} else if (QUEUE.endsWith(field.getTagName())
|| POOL.equals(field.getTagName())) {
loadQueue(queueName, field, builder);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
index ee5f179..f15f912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
@@ -52,6 +52,7 @@ public class QueueProperties {
reservationAcls;
private final Set<String> reservableQueues;
private final Set<String> nonPreemptableQueues;
+ private final Map<String, Boolean> queueOversubscriptionSetting;
private final Map<FSQueueType, Set<String>> configuredQueues;
QueueProperties(Builder builder) {
@@ -70,6 +71,7 @@ public class QueueProperties {
this.maxChildQueueResources = builder.maxChildQueueResources;
this.reservationAcls = builder.reservationAcls;
this.queueAcls = builder.queueAcls;
+ this.queueOversubscriptionSetting = builder.queueOversubscriptionSetting;
}
public Map<FSQueueType, Set<String>> getConfiguredQueues() {
@@ -133,6 +135,9 @@ public class QueueProperties {
return nonPreemptableQueues;
}
+ public Map<String, Boolean> getQueueOversubscriptionSetting() {
+ return queueOversubscriptionSetting;
+ }
/**
* Builder class for {@link QueueProperties}.
* All methods are adding queue properties to the maps of this builder
@@ -160,6 +165,8 @@ public class QueueProperties {
reservationAcls = new HashMap<>();
private Set<String> reservableQueues = new HashSet<>();
private Set<String> nonPreemptableQueues = new HashSet<>();
+ private Map<String, Boolean> queueOversubscriptionSetting =
+ new HashMap<>(0);
// Remember all queue names so we can display them on web UI, etc.
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
@@ -253,6 +260,12 @@ public class QueueProperties {
return this;
}
+ public Builder addQueueOversubscriptionSetting(
+ String queue, Boolean allow) {
+ this.queueOversubscriptionSetting.put(queue, allow);
+ return this;
+ }
+
public void configuredQueues(FSQueueType queueType, String queueName) {
this.configuredQueues.get(queueType).add(queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index fe313a6..45b9ea9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -68,6 +68,7 @@ public class FairSchedulerQueueInfo {
private String schedulingPolicy;
private boolean preemptable;
+ private boolean allowOversubscription;
private FairSchedulerQueueInfoList childQueues;
@@ -118,6 +119,7 @@ public class FairSchedulerQueueInfo {
}
preemptable = queue.isPreemptable();
+ allowOversubscription = queue.isOversubscriptionAllowed();
childQueues = getChildQueues(queue, scheduler);
}
@@ -255,4 +257,8 @@ public class FairSchedulerQueueInfo {
public boolean isPreemptable() {
return preemptable;
}
+
+ public boolean isOversubscriptionAllowed() {
+ return allowOversubscription;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 50a003e..35144cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -538,6 +538,67 @@ public class TestAllocationFileLoaderService {
}
@Test
+ public void testQueueOptOutOfOversubscription() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ // Queue A has the default value of allowOversubscription
+ out.println("<queue name=\"queueA\">");
+ // The child queue of A, queueA.child1 has overridden the parent's setting
+ out.println("<queue name=\"child1\">");
+ out.println("<allowOversubscription>false</allowOversubscription>");
+ out.println("</queue>");
+ // The child queue of A, queueA.child2 inherits the setting from its parent
+ out.println("<queue name=\"child2\" />");
+ out.println("</queue>");
+ // Queue A has opted out of oversubscription
+ out.println("<queue name=\"queueB\">");
+ // The child queue of B, queueB.child has overridden the parent's setting
+ out.println("<allowOversubscription>true</allowOversubscription>");
+ // The child queue of A, queueA.child1 has overridden the parent's setting
+ out.println("<queue name=\"child1\" />");
+ // The child queue of B, queueB.child2 inherits the setting from its parent
+ out.println("<queue name=\"child2\">");
+ out.println("<allowOversubscription>false</allowOversubscription>");
+ out.println("</queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ allocLoader.init(conf);
+ ReloadListener confHolder = new ReloadListener();
+ allocLoader.setReloadListener(confHolder);
+ allocLoader.reloadAllocations();
+ AllocationConfiguration queueConf = confHolder.allocConf;
+
+ assertTrue("allowOversubscription is not set for queueA",
+ !queueConf.isOversubscriptionAllowed("root.queueA").isPresent());
+ assertTrue("allowOversubscription is not set for queueA.child2",
+ !queueConf.isOversubscriptionAllowed("root.queueA.child2").isPresent());
+ assertTrue("allowOversubscription is not set for queueB.child1",
+ !queueConf.isOversubscriptionAllowed("root.queueB.child1").isPresent());
+
+ assertTrue("allowOversubscription is set for queueB",
+ queueConf.isOversubscriptionAllowed("root.queueB").isPresent());
+ assertTrue("queueB does allow oversubscription",
+ queueConf.isOversubscriptionAllowed("root.queueB").get());
+ assertTrue("allowOversubscription is set for queueB.child2",
+ queueConf.isOversubscriptionAllowed("root.queueB.child2").isPresent());
+ assertTrue("queueB.child2 does not allow oversubscription",
+ !queueConf.isOversubscriptionAllowed("root.queueB.child2").get());
+
+ assertTrue("allowOversubscription is set for queueA.child1",
+ queueConf.isOversubscriptionAllowed("root.queueA.child1").isPresent());
+ assertTrue("queueA.child1 does not allow oversubscription",
+ !queueConf.isOversubscriptionAllowed("root.queueA.child1").get());
+
+ }
+
+ @Test
public void testSimplePlacementPolicyFromConf() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 9302f88..5878ccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2789,6 +2789,109 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ @Test
+ public void testQueueOptOutOfOversubscription() throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue queue1 =
+ scheduler.getQueueManager().getLeafQueue("queue1", true);
+ // queue 2 has opted out of oversubscription
+ FSLeafQueue queue2 =
+ scheduler.getQueueManager().getLeafQueue("queue2", true);
+ queue2.allowOversubscription(false);
+
+ FSParentQueue queue3
+ = scheduler.getQueueManager().getParentQueue("queue3", true);
+ // queue 3 has opted out of oversubscription
+ queue3.allowOversubscription(false);
+ FSLeafQueue queue4 =
+ scheduler.getQueueManager().getLeafQueue("queue3.queue4", true);
+ assertTrue("queue 4 should inherit the setting from queue 3",
+ !queue4.isOversubscriptionAllowed());
+
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request in queue1 that leaves some unallocated
+ // resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request in queue 2 that has opted out of
+ // oversubscription.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+
+ // create another scheduling request in queue 4 that has opted out of
+ // oversubscription.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1536, "queue3.queue4", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3.queue4").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 3674ffb..10111f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -642,4 +642,74 @@ public class TestQueueManager {
assertNotNull("root.a does not exist", p);
}
+ /**
+ * Test queues created dynamically inherit the oversubscription setting
+ * from its parent queue, in which oversubscription is enabled.
+ */
+ @Test
+ public void testCreateQueueWithParentOversubscriptionSettingsEnabled() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q1 = queueManager.createQueue("root.queue1",
+ FSQueueType.PARENT);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+ assertTrue("root queue should always allow oversubscription",
+ queueManager.getParentQueue("root", false).isOversubscriptionAllowed());
+ assertTrue("root.queue1 should allow oversubscription as its parent",
+ queueManager.getParentQueue("root.queue1", false)
+ .isOversubscriptionAllowed());
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ assertTrue("root.queue1.queue2 should allow oversubscription as its parent",
+ queueManager.getLeafQueue("root.queue1.queue2", false)
+ .isOversubscriptionAllowed());
+
+ }
+
+ /**
+ * Test queues created dynamically inherit the oversubscription setting
+ * from its parent queue, in which oversubscription is disabled.
+ */
+ @Test
+ public void testCreateQueueWithParentOversubscriptionSettingsDisabled() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+ FSQueue q1 = queueManager.createQueue("root.queue1",
+ FSQueueType.PARENT);
+ q1.allowOversubscription(false);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+
+ assertTrue("root queue should always allow oversubscription",
+ queueManager.getParentQueue("root", false).isOversubscriptionAllowed());
+ assertTrue("root.queue1 should not allow oversubscription",
+ !queueManager.getParentQueue("root.queue1", false)
+ .isOversubscriptionAllowed());
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ assertTrue(
+ "root.queue1.queue2 should not allow oversubscription as its parent",
+ !queueManager.getLeafQueue("root.queue1.queue2", false)
+ .isOversubscriptionAllowed());
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index b5bcbf5..920b2ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -111,6 +111,7 @@ The allocation file must be in XML format. The format contains five types of ele
* **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is 0.5f.
* **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues.
+ * **allowOversubscription**: determines whether the scheduler is allowed to assign OPPORTUNISTIC resources to applications running in the queue when over-subscription is turned on. The default is true.
* **reservation**: indicates to the `ReservationSystem` that the queue's resources is available for users to reserve. This only applies for leaf queues. A leaf queue is not reservable if this property isn't configured.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-4511. Common scheduler
changes to support scheduler-specific oversubscription implementations.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 1836919..02c0cc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -563,7 +563,7 @@ public class TestNodeLabelContainerAllocation {
int numContainers) {
CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
SchedulerNode node = cs.getSchedulerNode(nodeId);
- Assert.assertEquals(numContainers, node.getNumContainers());
+ Assert.assertEquals(numContainers, node.getNumGuaranteedContainers());
}
/**
@@ -1065,7 +1065,7 @@ public class TestNodeLabelContainerAllocation {
for (int i = 0; i < 50; i++) {
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
- if (schedulerNode1.getNumContainers() == 0) {
+ if (schedulerNode1.getNumGuaranteedContainers() == 0) {
cycleWaited++;
}
}
@@ -1131,7 +1131,7 @@ public class TestNodeLabelContainerAllocation {
CSAMContainerLaunchDiagnosticsConstants.LAST_NODE_PROCESSED_MSG
+ nodeIdStr + " ( Partition : [x]"));
Assert.assertEquals(0, cs.getSchedulerNode(nm1.getNodeId())
- .getNumContainers());
+ .getNumGuaranteedContainers());
rm1.close();
}
@@ -1215,7 +1215,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x
- Assert.assertEquals(10, schedulerNode1.getNumContainers());
+ Assert.assertEquals(10, schedulerNode1.getNumGuaranteedContainers());
// check non-exclusive containers of LeafQueue is correctly updated
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
@@ -1943,7 +1943,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x
- Assert.assertEquals(5, schedulerNode1.getNumContainers());
+ Assert.assertEquals(5, schedulerNode1.getNumGuaranteedContainers());
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
@@ -2043,7 +2043,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x (non-exclusive)
- Assert.assertEquals(3, schedulerNode1.getNumContainers());
+ Assert.assertEquals(3, schedulerNode1.getNumGuaranteedContainers());
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
@@ -2074,7 +2074,7 @@ public class TestNodeLabelContainerAllocation {
cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
// app1 gets all resource in default partition
- Assert.assertEquals(2, schedulerNode2.getNumContainers());
+ Assert.assertEquals(2, schedulerNode2.getNumGuaranteedContainers());
// 3GB is used from label x quota. 2GB used from default label.
// So total 2.5 GB is remaining.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 2512787..6390297 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -20,16 +20,26 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +58,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
import org.junit.Before;
import org.junit.Test;
@@ -324,7 +336,9 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
for (FSSchedulerNode node : clusterNodeTracker.getAllNodes()) {
int i = ThreadLocalRandom.current().nextInt(-30, 30);
synchronized (scheduler) {
- node.deductUnallocatedResource(Resource.newInstance(i * 1024, i));
+ RMContainer container = createRMContainer(
+ node.getRMNode(), Resource.newInstance(i * 1024, 1));
+ node.allocateContainer(container);
}
}
}
@@ -338,6 +352,32 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
}
}
+ private static RMContainer createRMContainer(
+ RMNode node, Resource resource) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.
+ newInstance(ApplicationId.newInstance(0, 0), 0);
+ ContainerId cId =
+ ContainerId.newContainerId(appAttemptId, 0);
+ Container container = Container.newInstance(
+ cId, node.getNodeID(), node.getNodeAddress(), resource,
+ Priority.newInstance(0), null, ExecutionType.GUARANTEED);
+
+ Dispatcher dispatcher = new AsyncDispatcher();
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(dispatcher);
+ when(rmContext.getSystemMetricsPublisher()).
+ thenReturn(new NoOpSystemMetricPublisher());
+ when(rmContext.getYarnConfiguration()).
+ thenReturn(new YarnConfiguration());
+ when(rmContext.getContainerAllocationExpirer()).
+ thenReturn(new ContainerAllocationExpirer(dispatcher));
+ when(rmContext.getRMApplicationHistoryWriter()).
+ thenReturn(new RMApplicationHistoryWriter());
+ return new RMContainerImpl(container, null,
+ container.getId().getApplicationAttemptId(),
+ node.getNodeID(), "test", rmContext);
+ }
+
@Test
public void testFairSchedulerContinuousSchedulingInitTime() throws Exception {
scheduler.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 0e3d344..6726f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -87,7 +87,7 @@ public class TestFSSchedulerNode {
while (!Resources.isNone(schedulerNode.getUnallocatedResource())) {
createDefaultContainer();
schedulerNode.allocateContainer(containers.get(containers.size() - 1));
- schedulerNode.containerStarted(containers.get(containers.size() - 1).
+ schedulerNode.containerLaunched(containers.get(containers.size() - 1).
getContainerId());
}
}
@@ -183,9 +183,9 @@ public class TestFSSchedulerNode {
assertEquals("Nothing should have been allocated, yet",
Resources.none(), schedulerNode.getAllocatedResource());
schedulerNode.allocateContainer(containers.get(0));
- schedulerNode.containerStarted(containers.get(0).getContainerId());
+ schedulerNode.containerLaunched(containers.get(0).getContainerId());
schedulerNode.allocateContainer(containers.get(1));
- schedulerNode.containerStarted(containers.get(1).getContainerId());
+ schedulerNode.containerLaunched(containers.get(1).getContainerId());
schedulerNode.allocateContainer(containers.get(2));
assertEquals("Container should be allocated",
Resources.multiply(containers.get(0).getContainer().getResource(), 3.0),
@@ -225,7 +225,7 @@ public class TestFSSchedulerNode {
schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all remaining containers
@@ -266,7 +266,7 @@ public class TestFSSchedulerNode {
schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all remaining containers
@@ -312,7 +312,7 @@ public class TestFSSchedulerNode {
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -360,7 +360,7 @@ public class TestFSSchedulerNode {
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -399,7 +399,7 @@ public class TestFSSchedulerNode {
when(starvingApp.isStopped()).thenReturn(true);
allocateContainers(schedulerNode);
assertNotEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -437,7 +437,7 @@ public class TestFSSchedulerNode {
// Container partially reassigned
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- Resources.subtract(schedulerNode.getTotalResource(),
+ Resources.subtract(schedulerNode.getCapacity(),
Resource.newInstance(512, 0)),
schedulerNode.getAllocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 9120d3a..4b637ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3299,12 +3299,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(node2UpdateEvent);
if (invalid) {
assertEquals(0, app.getLiveContainers().size());
- assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
- assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ assertEquals(0,
+ scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+ assertEquals(0,
+ scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
} else {
assertEquals(1, app.getLiveContainers().size());
- assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
- assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ assertEquals(1,
+ scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+ assertEquals(0,
+ scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
}
}
@@ -5051,7 +5055,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Check total resource of scheduler node is also changed to 0 GB 0 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals(totalResource.getMemorySize(), 0 * GB);
Assert.assertEquals(totalResource.getVirtualCores(), 0);
// Check the available resource is 0/0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index ee66a49..3f8a552 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -344,7 +344,7 @@ public class TestFifoScheduler {
// SchedulerNode's total resource and available resource are changed.
assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID())
- .getTotalResource().getMemorySize());
+ .getCapacity().getMemorySize());
assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).
getUnallocatedResource().getMemorySize(), 1024);
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
@@ -1293,7 +1293,7 @@ public class TestFifoScheduler {
// Check total resource of scheduler node is also changed to 1 GB 1 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
Assert.assertEquals(totalResource.getVirtualCores(), 1);
// Check the available resource is 0/0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDDS-394. Rename *Key Apis in
DatanodeContainerProtocol to *Block apis. Contributed Dinesh Chitlangia.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index ed4536f..4f2b3a2 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.utils.MetadataKeyFilters;
import org.apache.hadoop.utils.MetadataStore;
@@ -116,7 +116,7 @@ public final class KeyValueContainerUtil {
File chunksPath = new File(containerData.getChunksPath());
// Close the DB connection and remove the DB handler from cache
- KeyUtils.removeDB(containerData, conf);
+ BlockUtils.removeDB(containerData, conf);
// Delete the Container MetaData path.
FileUtils.deleteDirectory(containerMetaDataPath);
@@ -175,16 +175,16 @@ public final class KeyValueContainerUtil {
}
kvContainerData.setDbFile(dbFile);
- MetadataStore metadata = KeyUtils.getDB(kvContainerData, config);
+ MetadataStore metadata = BlockUtils.getDB(kvContainerData, config);
long bytesUsed = 0;
List<Map.Entry<byte[], byte[]>> liveKeys = metadata
.getRangeKVs(null, Integer.MAX_VALUE,
MetadataKeyFilters.getNormalKeyFilter());
bytesUsed = liveKeys.parallelStream().mapToLong(e-> {
- KeyData keyData;
+ BlockData blockData;
try {
- keyData = KeyUtils.getKeyData(e.getValue());
- return keyData.getSize();
+ blockData = BlockUtils.getBlockData(e.getValue());
+ return blockData.getSize();
} catch (IOException ex) {
return 0L;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
index df60c60..3495363 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
@@ -69,7 +69,7 @@ public final class SmallFileUtils {
ContainerProtos.ReadChunkResponseProto.newBuilder();
readChunkresponse.setChunkData(info.getProtoBufMessage());
readChunkresponse.setData(ByteString.copyFrom(data));
- readChunkresponse.setBlockID(msg.getGetSmallFile().getKey().getBlockID());
+ readChunkresponse.setBlockID(msg.getGetSmallFile().getBlock().getBlockID());
ContainerProtos.GetSmallFileResponseProto.Builder getSmallFile =
ContainerProtos.GetSmallFileResponseProto.newBuilder();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
new file mode 100644
index 0000000..54c15fb
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
+import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
+import org.apache.hadoop.utils.MetadataStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK;
+
+/**
+ * This class is for performing block related operations on the KeyValue
+ * Container.
+ */
+public class BlockManagerImpl implements BlockManager {
+
+ static final Logger LOG = LoggerFactory.getLogger(BlockManagerImpl.class);
+
+ private Configuration config;
+
+ /**
+ * Constructs a Block Manager.
+ *
+ * @param conf - Ozone configuration
+ */
+ public BlockManagerImpl(Configuration conf) {
+ Preconditions.checkNotNull(conf, "Config cannot be null");
+ this.config = conf;
+ }
+
+ /**
+ * Puts or overwrites a block.
+ *
+ * @param container - Container for which block need to be added.
+ * @param data - BlockData.
+ * @return length of the block.
+ * @throws IOException
+ */
+ public long putBlock(Container container, BlockData data) throws IOException {
+ Preconditions.checkNotNull(data, "BlockData cannot be null for put " +
+ "operation.");
+ Preconditions.checkState(data.getContainerID() >= 0, "Container Id " +
+ "cannot be negative");
+ // We are not locking the key manager since LevelDb serializes all actions
+ // against a single DB. We rely on DB level locking to avoid conflicts.
+ MetadataStore db = BlockUtils.getDB((KeyValueContainerData) container
+ .getContainerData(), config);
+
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ db.put(Longs.toByteArray(data.getLocalID()), data.getProtoBufMessage()
+ .toByteArray());
+
+ // Increment keycount here
+ container.getContainerData().incrKeyCount();
+ return data.getSize();
+ }
+
+ /**
+ * Gets an existing block.
+ *
+ * @param container - Container from which block need to be fetched.
+ * @param blockID - BlockID of the block.
+ * @return Key Data.
+ * @throws IOException
+ */
+ public BlockData getBlock(Container container, BlockID blockID)
+ throws IOException {
+ Preconditions.checkNotNull(blockID,
+ "BlockID cannot be null in GetBlock request");
+ Preconditions.checkNotNull(blockID.getContainerID(),
+ "Container name cannot be null");
+
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = BlockUtils.getDB(containerData, config);
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID()));
+ if (kData == null) {
+ throw new StorageContainerException("Unable to find the block.",
+ NO_SUCH_BLOCK);
+ }
+ ContainerProtos.BlockData blockData =
+ ContainerProtos.BlockData.parseFrom(kData);
+ return BlockData.getFromProtoBuf(blockData);
+ }
+
+ /**
+ * Returns the length of the committed block.
+ *
+ * @param container - Container from which block need to be fetched.
+ * @param blockID - BlockID of the block.
+ * @return length of the block.
+ * @throws IOException in case, the block key does not exist in db.
+ */
+ @Override
+ public long getCommittedBlockLength(Container container, BlockID blockID)
+ throws IOException {
+ KeyValueContainerData containerData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = BlockUtils.getDB(containerData, config);
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID()));
+ if (kData == null) {
+ throw new StorageContainerException("Unable to find the block.",
+ NO_SUCH_BLOCK);
+ }
+ ContainerProtos.BlockData blockData =
+ ContainerProtos.BlockData.parseFrom(kData);
+ return blockData.getSize();
+ }
+
+ /**
+ * Deletes an existing block.
+ *
+ * @param container - Container from which block need to be deleted.
+ * @param blockID - ID of the block.
+ * @throws StorageContainerException
+ */
+ public void deleteBlock(Container container, BlockID blockID) throws
+ IOException {
+ Preconditions.checkNotNull(blockID, "block ID cannot be null.");
+ Preconditions.checkState(blockID.getContainerID() >= 0,
+ "Container ID cannot be negative.");
+ Preconditions.checkState(blockID.getLocalID() >= 0,
+ "Local ID cannot be negative.");
+
+ KeyValueContainerData cData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = BlockUtils.getDB(cData, config);
+ // This is a post condition that acts as a hint to the user.
+ // Should never fail.
+ Preconditions.checkNotNull(db, "DB cannot be null here");
+ // Note : There is a race condition here, since get and delete
+ // are not atomic. Leaving it here since the impact is refusing
+ // to delete a Block which might have just gotten inserted after
+ // the get check.
+ byte[] kKey = Longs.toByteArray(blockID.getLocalID());
+ byte[] kData = db.get(kKey);
+ if (kData == null) {
+ throw new StorageContainerException("Unable to find the block.",
+ NO_SUCH_BLOCK);
+ }
+ db.delete(kKey);
+
+ // Decrement blockcount here
+ container.getContainerData().decrKeyCount();
+ }
+
+ /**
+ * List blocks in a container.
+ *
+ * @param container - Container from which blocks need to be listed.
+ * @param startLocalID - Key to start from, 0 to begin.
+ * @param count - Number of blocks to return.
+ * @return List of Blocks that match the criteria.
+ */
+ @Override
+ public List<BlockData> listBlock(Container container, long startLocalID, int
+ count) throws IOException {
+ Preconditions.checkNotNull(container, "container cannot be null");
+ Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be " +
+ "negative");
+ Preconditions.checkArgument(count > 0,
+ "Count must be a positive number.");
+ container.readLock();
+ List<BlockData> result = null;
+ KeyValueContainerData cData = (KeyValueContainerData) container
+ .getContainerData();
+ MetadataStore db = BlockUtils.getDB(cData, config);
+ result = new ArrayList<>();
+ byte[] startKeyInBytes = Longs.toByteArray(startLocalID);
+ List<Map.Entry<byte[], byte[]>> range = db.getSequentialRangeKVs(
+ startKeyInBytes, count, null);
+ for (Map.Entry<byte[], byte[]> entry : range) {
+ BlockData value = BlockUtils.getBlockData(entry.getValue());
+ BlockData data = new BlockData(value.getBlockID());
+ result.add(data);
+ }
+ return result;
+ }
+
+ /**
+ * Shutdown KeyValueContainerManager.
+ */
+ public void shutdown() {
+ BlockUtils.shutdownCache(ContainerCache.getInstance(config));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java
deleted file mode 100644
index 6370f8e..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.keyvalue.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_KEY;
-
-/**
- * This class is for performing key related operations on the KeyValue
- * Container.
- */
-public class KeyManagerImpl implements KeyManager {
-
- static final Logger LOG = LoggerFactory.getLogger(KeyManagerImpl.class);
-
- private Configuration config;
-
- /**
- * Constructs a key Manager.
- *
- * @param conf - Ozone configuration
- */
- public KeyManagerImpl(Configuration conf) {
- Preconditions.checkNotNull(conf, "Config cannot be null");
- this.config = conf;
- }
-
- /**
- * Puts or overwrites a key.
- *
- * @param container - Container for which key need to be added.
- * @param data - Key Data.
- * @return length of the key.
- * @throws IOException
- */
- public long putKey(Container container, KeyData data) throws IOException {
- Preconditions.checkNotNull(data, "KeyData cannot be null for put " +
- "operation.");
- Preconditions.checkState(data.getContainerID() >= 0, "Container Id " +
- "cannot be negative");
- // We are not locking the key manager since LevelDb serializes all actions
- // against a single DB. We rely on DB level locking to avoid conflicts.
- MetadataStore db = KeyUtils.getDB((KeyValueContainerData) container
- .getContainerData(), config);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- db.put(Longs.toByteArray(data.getLocalID()), data.getProtoBufMessage()
- .toByteArray());
-
- // Increment keycount here
- container.getContainerData().incrKeyCount();
- return data.getSize();
- }
-
- /**
- * Gets an existing key.
- *
- * @param container - Container from which key need to be get.
- * @param blockID - BlockID of the key.
- * @return Key Data.
- * @throws IOException
- */
- public KeyData getKey(Container container, BlockID blockID)
- throws IOException {
- Preconditions.checkNotNull(blockID,
- "BlockID cannot be null in GetKet request");
- Preconditions.checkNotNull(blockID.getContainerID(),
- "Container name cannot be null");
-
- KeyValueContainerData containerData = (KeyValueContainerData) container
- .getContainerData();
- MetadataStore db = KeyUtils.getDB(containerData, config);
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID()));
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- ContainerProtos.KeyData keyData = ContainerProtos.KeyData.parseFrom(kData);
- return KeyData.getFromProtoBuf(keyData);
- }
-
- /**
- * Returns the length of the committed block.
- *
- * @param container - Container from which key need to be get.
- * @param blockID - BlockID of the key.
- * @return length of the block.
- * @throws IOException in case, the block key does not exist in db.
- */
- @Override
- public long getCommittedBlockLength(Container container, BlockID blockID)
- throws IOException {
- KeyValueContainerData containerData = (KeyValueContainerData) container
- .getContainerData();
- MetadataStore db = KeyUtils.getDB(containerData, config);
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID()));
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- ContainerProtos.KeyData keyData = ContainerProtos.KeyData.parseFrom(kData);
- return keyData.getSize();
- }
-
- /**
- * Deletes an existing Key.
- *
- * @param container - Container from which key need to be deleted.
- * @param blockID - ID of the block.
- * @throws StorageContainerException
- */
- public void deleteKey(Container container, BlockID blockID) throws
- IOException {
- Preconditions.checkNotNull(blockID, "block ID cannot be null.");
- Preconditions.checkState(blockID.getContainerID() >= 0,
- "Container ID cannot be negative.");
- Preconditions.checkState(blockID.getLocalID() >= 0,
- "Local ID cannot be negative.");
-
- KeyValueContainerData cData = (KeyValueContainerData) container
- .getContainerData();
- MetadataStore db = KeyUtils.getDB(cData, config);
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- // Note : There is a race condition here, since get and delete
- // are not atomic. Leaving it here since the impact is refusing
- // to delete a key which might have just gotten inserted after
- // the get check.
- byte[] kKey = Longs.toByteArray(blockID.getLocalID());
- byte[] kData = db.get(kKey);
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- db.delete(kKey);
-
- // Decrement keycount here
- container.getContainerData().decrKeyCount();
- }
-
- /**
- * List keys in a container.
- *
- * @param container - Container from which keys need to be listed.
- * @param startLocalID - Key to start from, 0 to begin.
- * @param count - Number of keys to return.
- * @return List of Keys that match the criteria.
- */
- @Override
- public List<KeyData> listKey(Container container, long startLocalID, int
- count) throws IOException {
- Preconditions.checkNotNull(container, "container cannot be null");
- Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be " +
- "negative");
- Preconditions.checkArgument(count > 0,
- "Count must be a positive number.");
- container.readLock();
- List<KeyData> result = null;
- KeyValueContainerData cData = (KeyValueContainerData) container
- .getContainerData();
- MetadataStore db = KeyUtils.getDB(cData, config);
- result = new ArrayList<>();
- byte[] startKeyInBytes = Longs.toByteArray(startLocalID);
- List<Map.Entry<byte[], byte[]>> range = db.getSequentialRangeKVs(
- startKeyInBytes, count, null);
- for (Map.Entry<byte[], byte[]> entry : range) {
- KeyData value = KeyUtils.getKeyData(entry.getValue());
- KeyData data = new KeyData(value.getBlockID());
- result.add(data);
- }
- return result;
- }
-
- /**
- * Shutdown KeyValueContainerManager.
- */
- public void shutdown() {
- KeyUtils.shutdownCache(ContainerCache.getInstance(config));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java
index 525d51b..564b50e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java
@@ -17,6 +17,5 @@
*/
package org.apache.hadoop.ozone.container.keyvalue.impl;
/**
- This package contains chunk manager and key manager implementation for
- keyvalue container type.
- **/
\ No newline at end of file
+ * Chunk manager and block manager implementations for keyvalue container type.
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
new file mode 100644
index 0000000..35ed22a
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.interfaces;
+
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * BlockManager is for performing key related operations on the container.
+ */
+public interface BlockManager {
+
+ /**
+ * Puts or overwrites a block.
+ *
+ * @param container - Container for which block need to be added.
+ * @param data - Block Data.
+ * @return length of the Block.
+ * @throws IOException
+ */
+ long putBlock(Container container, BlockData data) throws IOException;
+
+ /**
+ * Gets an existing block.
+ *
+ * @param container - Container from which block need to be get.
+ * @param blockID - BlockID of the Block.
+ * @return Block Data.
+ * @throws IOException
+ */
+ BlockData getBlock(Container container, BlockID blockID) throws IOException;
+
+ /**
+ * Deletes an existing block.
+ *
+ * @param container - Container from which block need to be deleted.
+ * @param blockID - ID of the block.
+ * @throws StorageContainerException
+ */
+ void deleteBlock(Container container, BlockID blockID) throws IOException;
+
+ /**
+ * List blocks in a container.
+ *
+ * @param container - Container from which blocks need to be listed.
+ * @param startLocalID - Block to start from, 0 to begin.
+ * @param count - Number of blocks to return.
+ * @return List of Blocks that match the criteria.
+ */
+ List<BlockData> listBlock(Container container, long startLocalID, int count)
+ throws IOException;
+
+ /**
+ * Returns the last committed block length for the block.
+ * @param blockID blockId
+ */
+ long getCommittedBlockLength(Container container, BlockID blockID)
+ throws IOException;
+
+ /**
+ * Shutdown ContainerManager.
+ */
+ void shutdown();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
deleted file mode 100644
index 84f771a..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.keyvalue.interfaces;
-
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KeyManager is for performing key related operations on the container.
- */
-public interface KeyManager {
-
- /**
- * Puts or overwrites a key.
- *
- * @param container - Container for which key need to be added.
- * @param data - Key Data.
- * @return length of the Key.
- * @throws IOException
- */
- long putKey(Container container, KeyData data) throws IOException;
-
- /**
- * Gets an existing key.
- *
- * @param container - Container from which key need to be get.
- * @param blockID - BlockID of the Key.
- * @return Key Data.
- * @throws IOException
- */
- KeyData getKey(Container container, BlockID blockID) throws IOException;
-
- /**
- * Deletes an existing Key.
- *
- * @param container - Container from which key need to be deleted.
- * @param blockID - ID of the block.
- * @throws StorageContainerException
- */
- void deleteKey(Container container, BlockID blockID) throws IOException;
-
- /**
- * List keys in a container.
- *
- * @param container - Container from which keys need to be listed.
- * @param startLocalID - Key to start from, 0 to begin.
- * @param count - Number of keys to return.
- * @return List of Keys that match the criteria.
- */
- List<KeyData> listKey(Container container, long startLocalID, int count)
- throws IOException;
-
- /**
- * Returns the last committed block length for the block.
- * @param blockID blockId
- */
- long getCommittedBlockLength(Container container, BlockID blockID)
- throws IOException;
-
- /**
- * Shutdown ContainerManager.
- */
- void shutdown();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java
new file mode 100644
index 0000000..5129094
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.keyvalue.interfaces;
+/**
+ * Chunk manager and block manager interfaces for keyvalue container type.
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index 51eed7f..d96fbfa 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.ratis.shaded.com.google.protobuf
.InvalidProtocolBufferException;
@@ -72,7 +72,7 @@ public class BlockDeletingService extends BackgroundService{
private static final Logger LOG =
LoggerFactory.getLogger(BlockDeletingService.class);
- ContainerSet containerSet;
+ private ContainerSet containerSet;
private ContainerDeletionChoosingPolicy containerDeletionPolicy;
private final Configuration conf;
@@ -185,7 +185,7 @@ public class BlockDeletingService extends BackgroundService{
ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
long startTime = Time.monotonicNow();
// Scan container's db and get list of under deletion blocks
- MetadataStore meta = KeyUtils.getDB(
+ MetadataStore meta = BlockUtils.getDB(
(KeyValueContainerData) containerData, conf);
// # of blocks to delete is throttled
KeyPrefixFilter filter =
@@ -211,8 +211,8 @@ public class BlockDeletingService extends BackgroundService{
String blockName = DFSUtil.bytes2String(entry.getKey());
LOG.debug("Deleting block {}", blockName);
try {
- ContainerProtos.KeyData data =
- ContainerProtos.KeyData.parseFrom(entry.getValue());
+ ContainerProtos.BlockData data =
+ ContainerProtos.BlockData.parseFrom(entry.getValue());
for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) {
File chunkFile = dataDir.toPath()
.resolve(chunkInfo.getChunkName()).toFile();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
new file mode 100644
index 0000000..6fe6d81
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue;
+
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.volume
+ .RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+
+/**
+ * This class is used to test key related operations on the container.
+ */
+public class TestBlockManagerImpl {
+
+ private OzoneConfiguration config;
+ private String scmId = UUID.randomUUID().toString();
+ private VolumeSet volumeSet;
+ private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
+ private KeyValueContainerData keyValueContainerData;
+ private KeyValueContainer keyValueContainer;
+ private BlockData blockData;
+ private BlockManagerImpl blockManager;
+ private BlockID blockID;
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+
+ @Before
+ public void setUp() throws Exception {
+ config = new OzoneConfiguration();
+
+ HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot()
+ .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID()
+ .toString()).build();
+
+ volumeSet = mock(VolumeSet.class);
+
+ volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class);
+ Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
+ .thenReturn(hddsVolume);
+
+ keyValueContainerData = new KeyValueContainerData(1L,
+ (long) StorageUnit.GB.toBytes(5));
+
+ keyValueContainer = new KeyValueContainer(
+ keyValueContainerData, config);
+
+ keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
+
+ // Creating BlockData
+ blockID = new BlockID(1L, 1L);
+ blockData = new BlockData(blockID);
+ blockData.addMetadata("VOLUME", "ozone");
+ blockData.addMetadata("OWNER", "hdfs");
+ List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
+ ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
+ .getLocalID(), 0), 0, 1024);
+ chunkList.add(info.getProtoBufMessage());
+ blockData.setChunks(chunkList);
+
+ // Create KeyValueContainerManager
+ blockManager = new BlockManagerImpl(config);
+
+ }
+
+ @Test
+ public void testPutAndGetBlock() throws Exception {
+ assertEquals(0, keyValueContainer.getContainerData().getKeyCount());
+ //Put Block
+ blockManager.putBlock(keyValueContainer, blockData);
+
+ assertEquals(1, keyValueContainer.getContainerData().getKeyCount());
+ //Get Block
+ BlockData fromGetBlockData = blockManager.getBlock(keyValueContainer,
+ blockData.getBlockID());
+
+ assertEquals(blockData.getContainerID(), fromGetBlockData.getContainerID());
+ assertEquals(blockData.getLocalID(), fromGetBlockData.getLocalID());
+ assertEquals(blockData.getChunks().size(),
+ fromGetBlockData.getChunks().size());
+ assertEquals(blockData.getMetadata().size(), fromGetBlockData.getMetadata()
+ .size());
+
+ }
+
+
+ @Test
+ public void testDeleteBlock() throws Exception {
+ try {
+ assertEquals(0,
+ keyValueContainer.getContainerData().getKeyCount());
+ //Put Block
+ blockManager.putBlock(keyValueContainer, blockData);
+ assertEquals(1,
+ keyValueContainer.getContainerData().getKeyCount());
+ //Delete Block
+ blockManager.deleteBlock(keyValueContainer, blockID);
+ assertEquals(0,
+ keyValueContainer.getContainerData().getKeyCount());
+ try {
+ blockManager.getBlock(keyValueContainer, blockID);
+ fail("testDeleteBlock");
+ } catch (StorageContainerException ex) {
+ GenericTestUtils.assertExceptionContains(
+ "Unable to find the block", ex);
+ }
+ } catch (IOException ex) {
+ fail("testDeleteBlock failed");
+ }
+ }
+
+ @Test
+ public void testListBlock() throws Exception {
+ try {
+ blockManager.putBlock(keyValueContainer, blockData);
+ List<BlockData> listBlockData = blockManager.listBlock(
+ keyValueContainer, 1, 10);
+ assertNotNull(listBlockData);
+ assertTrue(listBlockData.size() == 1);
+
+ for (long i = 2; i <= 10; i++) {
+ blockID = new BlockID(1L, i);
+ blockData = new BlockData(blockID);
+ blockData.addMetadata("VOLUME", "ozone");
+ blockData.addMetadata("OWNER", "hdfs");
+ List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
+ ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
+ .getLocalID(), 0), 0, 1024);
+ chunkList.add(info.getProtoBufMessage());
+ blockData.setChunks(chunkList);
+ blockManager.putBlock(keyValueContainer, blockData);
+ }
+
+ listBlockData = blockManager.listBlock(
+ keyValueContainer, 1, 10);
+ assertNotNull(listBlockData);
+ assertTrue(listBlockData.size() == 10);
+
+ } catch (IOException ex) {
+ fail("testListBlock failed");
+ }
+ }
+
+ @Test
+ public void testGetNoSuchBlock() throws Exception {
+ try {
+ assertEquals(0,
+ keyValueContainer.getContainerData().getKeyCount());
+ //Put Block
+ blockManager.putBlock(keyValueContainer, blockData);
+ assertEquals(1,
+ keyValueContainer.getContainerData().getKeyCount());
+ //Delete Block
+ blockManager.deleteBlock(keyValueContainer, blockID);
+ assertEquals(0,
+ keyValueContainer.getContainerData().getKeyCount());
+ try {
+ //Since the block has been deleted, we should not be able to find it
+ blockManager.getBlock(keyValueContainer, blockID);
+ fail("testGetNoSuchBlock failed");
+ } catch (StorageContainerException ex) {
+ GenericTestUtils.assertExceptionContains(
+ "Unable to find the block", ex);
+ assertEquals(ContainerProtos.Result.NO_SUCH_BLOCK, ex.getResult());
+ }
+ } catch (IOException ex) {
+ fail("testGetNoSuchBlock failed");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 9664052..3c0876b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -88,7 +88,7 @@ public class TestChunkManagerImpl {
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
data = "testing write chunks".getBytes();
- // Creating KeyData
+ // Creating BlockData
blockID = new BlockID(1L, 1L);
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, data.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
deleted file mode 100644
index b05dbca..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.keyvalue;
-
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.volume
- .RoundRobinVolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.UUID;
-
-import static org.junit.Assert.*;
-import static org.mockito.ArgumentMatchers.anyList;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.mock;
-
-/**
- * This class is used to test key related operations on the container.
- */
-public class TestKeyManagerImpl {
-
- private OzoneConfiguration config;
- private String scmId = UUID.randomUUID().toString();
- private VolumeSet volumeSet;
- private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
- private KeyValueContainerData keyValueContainerData;
- private KeyValueContainer keyValueContainer;
- private KeyData keyData;
- private KeyManagerImpl keyManager;
- private BlockID blockID;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
-
- @Before
- public void setUp() throws Exception {
- config = new OzoneConfiguration();
-
- HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot()
- .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID()
- .toString()).build();
-
- volumeSet = mock(VolumeSet.class);
-
- volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class);
- Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
- .thenReturn(hddsVolume);
-
- keyValueContainerData = new KeyValueContainerData(1L,
- (long) StorageUnit.GB.toBytes(5));
-
- keyValueContainer = new KeyValueContainer(
- keyValueContainerData, config);
-
- keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
-
- // Creating KeyData
- blockID = new BlockID(1L, 1L);
- keyData = new KeyData(blockID);
- keyData.addMetadata("VOLUME", "ozone");
- keyData.addMetadata("OWNER", "hdfs");
- List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
- ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
- .getLocalID(), 0), 0, 1024);
- chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
-
- // Create KeyValueContainerManager
- keyManager = new KeyManagerImpl(config);
-
- }
-
- @Test
- public void testPutAndGetKey() throws Exception {
- assertEquals(0, keyValueContainer.getContainerData().getKeyCount());
- //Put Key
- keyManager.putKey(keyValueContainer, keyData);
-
- assertEquals(1, keyValueContainer.getContainerData().getKeyCount());
- //Get Key
- KeyData fromGetKeyData = keyManager.getKey(keyValueContainer,
- keyData.getBlockID());
-
- assertEquals(keyData.getContainerID(), fromGetKeyData.getContainerID());
- assertEquals(keyData.getLocalID(), fromGetKeyData.getLocalID());
- assertEquals(keyData.getChunks().size(), fromGetKeyData.getChunks().size());
- assertEquals(keyData.getMetadata().size(), fromGetKeyData.getMetadata()
- .size());
-
- }
-
-
- @Test
- public void testDeleteKey() throws Exception {
- try {
- assertEquals(0, keyValueContainer.getContainerData().getKeyCount());
- //Put Key
- keyManager.putKey(keyValueContainer, keyData);
- assertEquals(1, keyValueContainer.getContainerData().getKeyCount());
- //Delete Key
- keyManager.deleteKey(keyValueContainer, blockID);
- assertEquals(0, keyValueContainer.getContainerData().getKeyCount());
- try {
- keyManager.getKey(keyValueContainer, blockID);
- fail("testDeleteKey");
- } catch (StorageContainerException ex) {
- GenericTestUtils.assertExceptionContains("Unable to find the key", ex);
- }
- } catch (IOException ex) {
- fail("testDeleteKey failed");
- }
- }
-
- @Test
- public void testListKey() throws Exception {
- try {
- keyManager.putKey(keyValueContainer, keyData);
- List<KeyData> listKeyData = keyManager.listKey(
- keyValueContainer, 1, 10);
- assertNotNull(listKeyData);
- assertTrue(listKeyData.size() == 1);
-
- for (long i = 2; i <= 10; i++) {
- blockID = new BlockID(1L, i);
- keyData = new KeyData(blockID);
- keyData.addMetadata("VOLUME", "ozone");
- keyData.addMetadata("OWNER", "hdfs");
- List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
- ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
- .getLocalID(), 0), 0, 1024);
- chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
- keyManager.putKey(keyValueContainer, keyData);
- }
-
- listKeyData = keyManager.listKey(
- keyValueContainer, 1, 10);
- assertNotNull(listKeyData);
- assertTrue(listKeyData.size() == 10);
-
- } catch (IOException ex) {
- fail("testListKey failed");
- }
- }
-
- @Test
- public void testGetNoSuchKey() throws Exception {
- try {
- keyData = new KeyData(new BlockID(1L, 2L));
- keyManager.getKey(keyValueContainer, new BlockID(1L, 2L));
- fail("testGetNoSuchKey failed");
- } catch (StorageContainerException ex) {
- GenericTestUtils.assertExceptionContains("Unable to find the key.", ex);
- assertEquals(ContainerProtos.Result.NO_SUCH_KEY, ex.getResult());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
index f1fe88e..fbc5ad0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
@@ -27,11 +27,11 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.utils.MetadataKeyFilters;
import org.apache.hadoop.utils.MetadataStore;
@@ -114,8 +114,8 @@ public class TestKeyValueBlockIterator {
int counter = 0;
while(keyValueBlockIterator.hasNext()) {
- KeyData keyData = keyValueBlockIterator.nextBlock();
- assertEquals(keyData.getLocalID(), counter++);
+ BlockData blockData = keyValueBlockIterator.nextBlock();
+ assertEquals(blockData.getLocalID(), counter++);
}
assertFalse(keyValueBlockIterator.hasNext());
@@ -123,8 +123,8 @@ public class TestKeyValueBlockIterator {
keyValueBlockIterator.seekToFirst();
counter = 0;
while(keyValueBlockIterator.hasNext()) {
- KeyData keyData = keyValueBlockIterator.nextBlock();
- assertEquals(keyData.getLocalID(), counter++);
+ BlockData blockData = keyValueBlockIterator.nextBlock();
+ assertEquals(blockData.getLocalID(), counter++);
}
assertFalse(keyValueBlockIterator.hasNext());
@@ -214,8 +214,8 @@ public class TestKeyValueBlockIterator {
int counter = 5;
while(keyValueBlockIterator.hasNext()) {
- KeyData keyData = keyValueBlockIterator.nextBlock();
- assertEquals(keyData.getLocalID(), counter++);
+ BlockData blockData = keyValueBlockIterator.nextBlock();
+ assertEquals(blockData.getLocalID(), counter++);
}
}
@@ -250,7 +250,7 @@ public class TestKeyValueBlockIterator {
container = new KeyValueContainer(containerData, conf);
container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID
.randomUUID().toString());
- MetadataStore metadataStore = KeyUtils.getDB(containerData, conf);
+ MetadataStore metadataStore = BlockUtils.getDB(containerData, conf);
List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
ChunkInfo info = new ChunkInfo("chunkfile", 0, 1024);
@@ -258,18 +258,18 @@ public class TestKeyValueBlockIterator {
for (int i=0; i<normalBlocks; i++) {
BlockID blockID = new BlockID(containerId, i);
- KeyData keyData = new KeyData(blockID);
- keyData.setChunks(chunkList);
- metadataStore.put(Longs.toByteArray(blockID.getLocalID()), keyData
+ BlockData blockData = new BlockData(blockID);
+ blockData.setChunks(chunkList);
+ metadataStore.put(Longs.toByteArray(blockID.getLocalID()), blockData
.getProtoBufMessage().toByteArray());
}
for (int i=normalBlocks; i<deletedBlocks; i++) {
BlockID blockID = new BlockID(containerId, i);
- KeyData keyData = new KeyData(blockID);
- keyData.setChunks(chunkList);
+ BlockData blockData = new BlockData(blockID);
+ blockData.setChunks(chunkList);
metadataStore.put(DFSUtil.string2Bytes(OzoneConsts
- .DELETING_KEY_PREFIX + blockID.getLocalID()), keyData
+ .DELETING_KEY_PREFIX + blockID.getLocalID()), blockData
.getProtoBufMessage().toByteArray());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index f84ba7d..bf6b8b0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -28,14 +28,14 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerLifeCycleState;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume
.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.utils.MetadataStore;
@@ -117,11 +117,11 @@ public class TestKeyValueContainer {
addBlocks(blockCount);
blockIterator = keyValueContainer.blockIterator();
assertTrue(blockIterator.hasNext());
- KeyData keyData;
+ BlockData blockData;
int blockCounter = 0;
while(blockIterator.hasNext()) {
- keyData = blockIterator.nextBlock();
- assertEquals(blockCounter++, keyData.getBlockID().getLocalID());
+ blockData = blockIterator.nextBlock();
+ assertEquals(blockCounter++, blockData.getBlockID().getLocalID());
}
assertEquals(blockCount, blockCounter);
}
@@ -129,20 +129,20 @@ public class TestKeyValueContainer {
private void addBlocks(int count) throws Exception {
long containerId = keyValueContainerData.getContainerID();
- MetadataStore metadataStore = KeyUtils.getDB(keyValueContainer
+ MetadataStore metadataStore = BlockUtils.getDB(keyValueContainer
.getContainerData(), conf);
for (int i=0; i < count; i++) {
- // Creating KeyData
+ // Creating BlockData
BlockID blockID = new BlockID(containerId, i);
- KeyData keyData = new KeyData(blockID);
- keyData.addMetadata("VOLUME", "ozone");
- keyData.addMetadata("OWNER", "hdfs");
+ BlockData blockData = new BlockData(blockID);
+ blockData.addMetadata("VOLUME", "ozone");
+ blockData.addMetadata("OWNER", "hdfs");
List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, 1024);
chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
- metadataStore.put(Longs.toByteArray(blockID.getLocalID()), keyData
+ blockData.setChunks(chunkList);
+ metadataStore.put(Longs.toByteArray(blockID.getLocalID()), blockData
.getProtoBufMessage().toByteArray());
}
@@ -189,7 +189,7 @@ public class TestKeyValueContainer {
int numberOfKeysToWrite = 12;
//write one few keys to check the key count after import
- MetadataStore metadataStore = KeyUtils.getDB(keyValueContainerData, conf);
+ MetadataStore metadataStore = BlockUtils.getDB(keyValueContainerData, conf);
for (int i = 0; i < numberOfKeysToWrite; i++) {
metadataStore.put(("test" + i).getBytes(), "test".getBytes());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index d91bbf7..e1904c1 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -142,31 +142,31 @@ public class TestKeyValueHandler {
Mockito.verify(handler, times(1)).handleCloseContainer(
any(ContainerCommandRequestProto.class), any());
- // Test Put Key Request handling
- ContainerCommandRequestProto putKeyRequest =
- getDummyCommandRequestProto(ContainerProtos.Type.PutKey);
- dispatcher.dispatch(putKeyRequest);
- Mockito.verify(handler, times(1)).handlePutKey(
+ // Test Put Block Request handling
+ ContainerCommandRequestProto putBlockRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.PutBlock);
+ dispatcher.dispatch(putBlockRequest);
+ Mockito.verify(handler, times(1)).handlePutBlock(
any(ContainerCommandRequestProto.class), any());
- // Test Get Key Request handling
- ContainerCommandRequestProto getKeyRequest =
- getDummyCommandRequestProto(ContainerProtos.Type.GetKey);
- dispatcher.dispatch(getKeyRequest);
- Mockito.verify(handler, times(1)).handleGetKey(
+ // Test Get Block Request handling
+ ContainerCommandRequestProto getBlockRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.GetBlock);
+ dispatcher.dispatch(getBlockRequest);
+ Mockito.verify(handler, times(1)).handleGetBlock(
any(ContainerCommandRequestProto.class), any());
- // Test Delete Key Request handling
- ContainerCommandRequestProto deleteKeyRequest =
- getDummyCommandRequestProto(ContainerProtos.Type.DeleteKey);
- dispatcher.dispatch(deleteKeyRequest);
- Mockito.verify(handler, times(1)).handleDeleteKey(
+ // Test Delete Block Request handling
+ ContainerCommandRequestProto deleteBlockRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.DeleteBlock);
+ dispatcher.dispatch(deleteBlockRequest);
+ Mockito.verify(handler, times(1)).handleDeleteBlock(
any(ContainerCommandRequestProto.class), any());
- // Test List Key Request handling
- ContainerCommandRequestProto listKeyRequest =
- getDummyCommandRequestProto(ContainerProtos.Type.ListKey);
- dispatcher.dispatch(listKeyRequest);
+ // Test List Block Request handling
+ ContainerCommandRequestProto listBlockRequest =
+ getDummyCommandRequestProto(ContainerProtos.Type.ListBlock);
+ dispatcher.dispatch(listBlockRequest);
Mockito.verify(handler, times(2)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
index 94966f6..2b10578 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
@@ -281,10 +281,10 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
groupInputStream.streamOffset[i] = length;
ContainerProtos.DatanodeBlockID datanodeBlockID = blockID
.getDatanodeBlockIDProtobuf();
- ContainerProtos.GetKeyResponseProto response = ContainerProtocolCalls
- .getKey(xceiverClient, datanodeBlockID, requestId);
+ ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls
+ .getBlock(xceiverClient, datanodeBlockID, requestId);
List<ContainerProtos.ChunkInfo> chunks =
- response.getKeyData().getChunksList();
+ response.getBlockData().getChunksList();
for (ContainerProtos.ChunkInfo chunk : chunks) {
length += chunk.getLen();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index d0e173c..0537f8a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -165,10 +165,10 @@ public class TestStorageContainerManagerHelper {
DatanodeDetails leadDN = containerWithPipeline.getPipeline().getLeader();
OzoneContainer containerServer =
getContainerServerByDatanodeUuid(leadDN.getUuidString());
- KeyValueContainerData containerData = (KeyValueContainerData) containerServer
- .getContainerSet()
+ KeyValueContainerData containerData =
+ (KeyValueContainerData) containerServer.getContainerSet()
.getContainer(containerID).getContainerData();
- return KeyUtils.getDB(containerData, conf);
+ return BlockUtils.getDB(containerData, conf);
}
private OzoneContainer getContainerServerByDatanodeUuid(String dnUUID)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java
index ddff0c5..b4a0ba7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.junit.AfterClass;
@@ -469,9 +469,9 @@ public class TestOzoneRestClient {
containerID, new File(containerPath));
long valueLength = 0;
while (keyValueBlockIterator.hasNext()) {
- KeyData keyData = keyValueBlockIterator.nextBlock();
- if (keyData.getBlockID().getLocalID() == localID) {
- List<ContainerProtos.ChunkInfo> chunks = keyData.getChunks();
+ BlockData blockData = keyValueBlockIterator.nextBlock();
+ if (blockData.getBlockID().getLocalID() == localID) {
+ List<ContainerProtos.ChunkInfo> chunks = blockData.getChunks();
for (ContainerProtos.ChunkInfo chunk : chunks) {
valueLength += chunk.getLen();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
index bf1eba6..cc045d0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -603,10 +603,10 @@ public class TestOzoneRpcClient {
KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator(
containerID, new File(containerPath));
while (keyValueBlockIterator.hasNext()) {
- KeyData keyData = keyValueBlockIterator.nextBlock();
- if (keyData.getBlockID().getLocalID() == localID) {
+ BlockData blockData = keyValueBlockIterator.nextBlock();
+ if (blockData.getBlockID().getLocalID() == localID) {
long length = 0;
- List<ContainerProtos.ChunkInfo> chunks = keyData.getChunks();
+ List<ContainerProtos.ChunkInfo> chunks = blockData.getChunks();
for (ContainerProtos.ChunkInfo chunk : chunks) {
length += chunk.getLen();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index 0c86828..f278479 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.util.Time;
import org.junit.Assert;
@@ -241,18 +241,18 @@ public final class ContainerTestHelper {
setDataChecksum(info, data);
- ContainerProtos.PutKeyRequestProto.Builder putRequest =
- ContainerProtos.PutKeyRequestProto.newBuilder();
+ ContainerProtos.PutBlockRequestProto.Builder putRequest =
+ ContainerProtos.PutBlockRequestProto.newBuilder();
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> newList = new LinkedList<>();
newList.add(info.getProtoBufMessage());
- keyData.setChunks(newList);
- putRequest.setKeyData(keyData.getProtoBufMessage());
+ blockData.setChunks(newList);
+ putRequest.setBlockData(blockData.getProtoBufMessage());
smallFileRequest.setChunkInfo(info.getProtoBufMessage());
smallFileRequest.setData(ByteString.copyFrom(data));
- smallFileRequest.setKey(putRequest);
+ smallFileRequest.setBlock(putRequest);
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
@@ -266,17 +266,17 @@ public final class ContainerTestHelper {
public static ContainerCommandRequestProto getReadSmallFileRequest(
- Pipeline pipeline, ContainerProtos.PutKeyRequestProto putKey)
+ Pipeline pipeline, ContainerProtos.PutBlockRequestProto putKey)
throws Exception {
ContainerProtos.GetSmallFileRequestProto.Builder smallFileRequest =
ContainerProtos.GetSmallFileRequestProto.newBuilder();
- ContainerCommandRequestProto getKey = getKeyRequest(pipeline, putKey);
- smallFileRequest.setKey(getKey.getGetKey());
+ ContainerCommandRequestProto getKey = getBlockRequest(pipeline, putKey);
+ smallFileRequest.setBlock(getKey.getGetBlock());
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
request.setCmdType(ContainerProtos.Type.GetSmallFile);
- request.setContainerID(getKey.getGetKey().getBlockID().getContainerID());
+ request.setContainerID(getKey.getGetBlock().getBlockID().getContainerID());
request.setGetSmallFile(smallFileRequest);
request.setTraceID(UUID.randomUUID().toString());
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
@@ -421,58 +421,58 @@ public final class ContainerTestHelper {
}
/**
- * Returns the PutKeyRequest for test purpose.
+ * Returns the PutBlockRequest for test purpose.
* @param pipeline - pipeline.
* @param writeRequest - Write Chunk Request.
* @return - Request
*/
- public static ContainerCommandRequestProto getPutKeyRequest(
+ public static ContainerCommandRequestProto getPutBlockRequest(
Pipeline pipeline, ContainerProtos.WriteChunkRequestProto writeRequest) {
- LOG.trace("putKey: {} to pipeline={}",
+ LOG.trace("putBlock: {} to pipeline={}",
writeRequest.getBlockID());
- ContainerProtos.PutKeyRequestProto.Builder putRequest =
- ContainerProtos.PutKeyRequestProto.newBuilder();
+ ContainerProtos.PutBlockRequestProto.Builder putRequest =
+ ContainerProtos.PutBlockRequestProto.newBuilder();
- KeyData keyData = new KeyData(
+ BlockData blockData = new BlockData(
BlockID.getFromProtobuf(writeRequest.getBlockID()));
List<ContainerProtos.ChunkInfo> newList = new LinkedList<>();
newList.add(writeRequest.getChunkData());
- keyData.setChunks(newList);
- putRequest.setKeyData(keyData.getProtoBufMessage());
+ blockData.setChunks(newList);
+ putRequest.setBlockData(blockData.getProtoBufMessage());
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.PutKey);
- request.setContainerID(keyData.getContainerID());
- request.setPutKey(putRequest);
+ request.setCmdType(ContainerProtos.Type.PutBlock);
+ request.setContainerID(blockData.getContainerID());
+ request.setPutBlock(putRequest);
request.setTraceID(UUID.randomUUID().toString());
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
return request.build();
}
/**
- * Gets a GetKeyRequest for test purpose.
+ * Gets a GetBlockRequest for test purpose.
* @param pipeline - pipeline
- * @param putKeyRequest - putKeyRequest.
+ * @param putBlockRequest - putBlockRequest.
* @return - Request
* immediately.
*/
- public static ContainerCommandRequestProto getKeyRequest(
- Pipeline pipeline, ContainerProtos.PutKeyRequestProto putKeyRequest) {
+ public static ContainerCommandRequestProto getBlockRequest(
+ Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest) {
ContainerProtos.DatanodeBlockID blockID =
- putKeyRequest.getKeyData().getBlockID();
+ putBlockRequest.getBlockData().getBlockID();
LOG.trace("getKey: blockID={}", blockID);
- ContainerProtos.GetKeyRequestProto.Builder getRequest =
- ContainerProtos.GetKeyRequestProto.newBuilder();
+ ContainerProtos.GetBlockRequestProto.Builder getRequest =
+ ContainerProtos.GetBlockRequestProto.newBuilder();
getRequest.setBlockID(blockID);
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.GetKey);
+ request.setCmdType(ContainerProtos.Type.GetBlock);
request.setContainerID(blockID.getContainerID());
- request.setGetKey(getRequest);
+ request.setGetBlock(getRequest);
request.setTraceID(UUID.randomUUID().toString());
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
return request.build();
@@ -484,32 +484,32 @@ public final class ContainerTestHelper {
* @param request - Request
* @param response - Response
*/
- public static void verifyGetKey(ContainerCommandRequestProto request,
+ public static void verifyGetBlock(ContainerCommandRequestProto request,
ContainerCommandResponseProto response, int expectedChunksCount) {
Assert.assertEquals(request.getTraceID(), response.getTraceID());
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
Assert.assertEquals(expectedChunksCount,
- response.getGetKey().getKeyData().getChunksCount());
+ response.getGetBlock().getBlockData().getChunksCount());
}
/**
* @param pipeline - pipeline.
- * @param putKeyRequest - putKeyRequest.
+ * @param putBlockRequest - putBlockRequest.
* @return - Request
*/
- public static ContainerCommandRequestProto getDeleteKeyRequest(
- Pipeline pipeline, ContainerProtos.PutKeyRequestProto putKeyRequest) {
- ContainerProtos.DatanodeBlockID blockID = putKeyRequest.getKeyData()
+ public static ContainerCommandRequestProto getDeleteBlockRequest(
+ Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest) {
+ ContainerProtos.DatanodeBlockID blockID = putBlockRequest.getBlockData()
.getBlockID();
- LOG.trace("deleteKey: name={}", blockID);
- ContainerProtos.DeleteKeyRequestProto.Builder delRequest =
- ContainerProtos.DeleteKeyRequestProto.newBuilder();
+ LOG.trace("deleteBlock: name={}", blockID);
+ ContainerProtos.DeleteBlockRequestProto.Builder delRequest =
+ ContainerProtos.DeleteBlockRequestProto.newBuilder();
delRequest.setBlockID(blockID);
ContainerCommandRequestProto.Builder request =
ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.DeleteKey);
+ request.setCmdType(ContainerProtos.Type.DeleteBlock);
request.setContainerID(blockID.getContainerID());
- request.setDeleteKey(delRequest);
+ request.setDeleteBlock(delRequest);
request.setTraceID(UUID.randomUUID().toString());
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
return request.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
index 7391b25..52cebb3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
@@ -100,17 +100,20 @@ public class TestContainerReplication {
DatanodeBlockID blockID = requestProto.getWriteChunk().getBlockID();
- // Put Key to the test container
- ContainerCommandRequestProto putKeyRequest = ContainerTestHelper
- .getPutKeyRequest(sourcePipelines, requestProto.getWriteChunk());
+ // Put Block to the test container
+ ContainerCommandRequestProto putBlockRequest = ContainerTestHelper
+ .getPutBlockRequest(sourcePipelines, requestProto.getWriteChunk());
- ContainerProtos.KeyData keyData = putKeyRequest.getPutKey().getKeyData();
+ ContainerProtos.BlockData blockData =
+ putBlockRequest.getPutBlock().getBlockData();
- ContainerCommandResponseProto response = client.sendCommand(putKeyRequest);
+ ContainerCommandResponseProto response =
+ client.sendCommand(putBlockRequest);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
- Assert.assertTrue(putKeyRequest.getTraceID().equals(response.getTraceID()));
+ Assert.assertTrue(
+ putBlockRequest.getTraceID().equals(response.getTraceID()));
HddsDatanodeService destinationDatanode =
chooseDatanodeWithoutContainer(sourcePipelines,
@@ -147,8 +150,8 @@ public class TestContainerReplication {
KeyValueHandler handler = (KeyValueHandler) ozoneContainer.getDispatcher()
.getHandler(ContainerType.KeyValueContainer);
- KeyData key = handler.getKeyManager()
- .getKey(container, BlockID.getFromProtobuf(blockID));
+ BlockData key = handler.getBlockManager()
+ .getBlock(container, BlockID.getFromProtobuf(blockID));
Assert.assertNotNull(key);
Assert.assertEquals(1, key.getChunks().size());
@@ -164,7 +167,8 @@ public class TestContainerReplication {
return datanode;
}
}
- throw new AssertionError("No datanode outside of the pipeline");
+ throw new AssertionError(
+ "No datanode outside of the pipeline");
}
static OzoneConfiguration newOzoneConfiguration() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 25c8c6b..7e30c5f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
@@ -33,9 +34,8 @@ import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingP
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.testutils.BlockDeletingServiceTestImpl;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background
.BlockDeletingService;
@@ -117,13 +117,13 @@ public class TestBlockDeletingService {
containerSet.addContainer(container);
data = (KeyValueContainerData) containerSet.getContainer(
containerID).getContainerData();
- MetadataStore metadata = KeyUtils.getDB(data, conf);
+ MetadataStore metadata = BlockUtils.getDB(data, conf);
for (int j = 0; j<numOfBlocksPerContainer; j++) {
BlockID blockID =
ContainerTestHelper.getTestBlockID(containerID);
String deleteStateName = OzoneConsts.DELETING_KEY_PREFIX +
blockID.getLocalID();
- KeyData kd = new KeyData(blockID);
+ BlockData kd = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList();
for (int k = 0; k<numOfChunksPerBlock; k++) {
// offset doesn't matter here
@@ -200,7 +200,7 @@ public class TestBlockDeletingService {
containerSet.listContainer(0L, 1, containerData);
Assert.assertEquals(1, containerData.size());
- MetadataStore meta = KeyUtils.getDB(
+ MetadataStore meta = BlockUtils.getDB(
(KeyValueContainerData) containerData.get(0), conf);
Map<Long, Container> containerMap = containerSet.getContainerMap();
// NOTE: this test assumes that all the container is KetValueContainer and
@@ -309,7 +309,7 @@ public class TestBlockDeletingService {
// get container meta data
List<ContainerData> containerData = Lists.newArrayList();
containerSet.listContainer(0L, 1, containerData);
- MetadataStore meta = KeyUtils.getDB(
+ MetadataStore meta = BlockUtils.getDB(
(KeyValueContainerData) containerData.get(0), conf);
LogCapturer newLog = LogCapturer.captureLogs(BackgroundService.LOG);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-15772. Remove the 'Path ...
should be specified as a URI' warnings on startup. Contributed by Ayush
Saxena.
Posted by ha...@apache.org.
HADOOP-15772. Remove the 'Path ... should be specified as a URI' warnings on startup. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2eb597b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2eb597b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2eb597b1
Branch: refs/heads/YARN-1011
Commit: 2eb597b1511f8f46866abe4eeec820f4191cc295
Parents: a30b4f9
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Sep 19 13:48:27 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Sep 19 13:48:27 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/server/common/Util.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2eb597b1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index 4e30e50..e9f9bfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -97,8 +97,7 @@ public final class Util {
// if URI is null or scheme is undefined, then assume it's file://
if(u == null || u.getScheme() == null){
- LOG.warn("Path " + s + " should be specified as a URI "
- + "in configuration files. Please update hdfs configuration.");
+ LOG.info("Assuming 'file' scheme for path " + s + " in configuration.");
u = fileAsURI(new File(s));
}
return u;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDDS-509.
TestStorageContainerManager is flaky. Contributed by Xiaoyu Yao and Ajay
Kumar.
Posted by ha...@apache.org.
HDDS-509. TestStorageContainerManager is flaky.
Contributed by Xiaoyu Yao and Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98c9bc4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98c9bc4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98c9bc4a
Branch: refs/heads/YARN-1011
Commit: 98c9bc4aa0ef6358e1ea6ca2f18cec7ae24158dd
Parents: 090272d
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 19 12:57:44 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 19 13:03:19 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/TestStorageContainerManager.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98c9bc4a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 3d9a043..94ab6c8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -588,11 +588,10 @@ public class TestStorageContainerManager {
helper.createKeys(10, 4096);
SCMClientProtocolServer clientProtocolServer = cluster
.getStorageContainerManager().getClientProtocolServer();
-
+ assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
final List<ContainerInfo> containers = scm.getScmContainerManager()
.getStateManager().getAllContainers();
scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
- assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
GenericTestUtils.waitFor(() -> {
return clientProtocolServer.getChillModeStatus();
}, 50, 1000 * 5);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: Revert "Revert "YARN-6675. Add NM
support to launch opportunistic containers based on overallocation.
Contributed by Haibo Chen.""
Posted by ha...@apache.org.
Revert "Revert "YARN-6675. Add NM support to launch opportunistic containers based on overallocation. Contributed by Haibo Chen.""
This reverts commit 1235cf5deb9f309e975c4f59ee27caf1282f19f9.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/faa71dbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/faa71dbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/faa71dbc
Branch: refs/heads/YARN-1011
Commit: faa71dbcd14f5b0dec2dd71f0c84bd9479157376
Parents: 23ae827
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jun 11 13:34:40 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:35:12 2018 -0700
----------------------------------------------------------------------
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 8 +-
.../launcher/ContainerLaunch.java | 2 +-
.../launcher/ContainersLauncher.java | 9 +-
.../monitor/ContainersMonitor.java | 38 +-
.../monitor/ContainersMonitorImpl.java | 56 +-
.../AllocationBasedResourceTracker.java | 114 ++
...locationBasedResourceUtilizationTracker.java | 158 ---
.../scheduler/ContainerScheduler.java | 317 +++--
.../scheduler/ContainerSchedulerEventType.java | 4 +-
.../scheduler/NMAllocationPolicy.java | 63 +
.../scheduler/ResourceUtilizationTracker.java | 17 +-
.../SnapshotBasedOverAllocationPolicy.java | 54 +
.../UtilizationBasedResourceTracker.java | 95 ++
.../BaseContainerManagerTest.java | 35 +
.../TestContainersMonitorResourceChange.java | 9 +-
.../TestAllocationBasedResourceTracker.java | 82 ++
...locationBasedResourceUtilizationTracker.java | 93 --
.../TestContainerSchedulerRecovery.java | 58 +-
...estContainerSchedulerWithOverAllocation.java | 1121 ++++++++++++++++++
20 files changed, 1916 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 572684e..d757376 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -543,7 +543,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private ResourceUtilization getContainersUtilization() {
ContainersMonitor containersMonitor =
this.context.getContainerManager().getContainersMonitor();
- return containersMonitor.getContainersUtilization();
+ return containersMonitor.getContainersUtilization(false).getUtilization();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 27a7c80..a08e227 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -245,6 +245,12 @@ public class ContainerManagerImpl extends CompositeService implements
metrics);
addService(rsrcLocalizationSrvc);
+ this.containersMonitor = createContainersMonitor(exec);
+ addService(this.containersMonitor);
+
+ // ContainersLauncher must be added after ContainersMonitor
+ // because the former depends on the latter to initialize
+ // over-allocation first.
containersLauncher = createContainersLauncher(context, exec);
addService(containersLauncher);
@@ -269,8 +275,6 @@ public class ContainerManagerImpl extends CompositeService implements
nmMetricsPublisher = createNMTimelinePublisher(context);
context.setNMTimelinePublisher(nmMetricsPublisher);
}
- this.containersMonitor = createContainersMonitor(exec);
- addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
new ContainerEventDispatcher());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6347d4e..0228332 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1079,7 +1079,7 @@ public class ContainerLaunch implements Callable<Integer> {
* @return Process ID
* @throws Exception
*/
- private String getContainerPid(Path pidFilePath) throws Exception {
+ protected String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 7870f86..2f5acfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -121,8 +121,7 @@ public class ContainersLauncher extends AbstractService
containerId.getApplicationAttemptId().getApplicationId());
ContainerLaunch launch =
- new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
- event.getContainer(), dirsHandler, containerManager);
+ createContainerLaunch(app, event.getContainer());
containerLauncher.submit(launch);
running.put(containerId, launch);
break;
@@ -225,4 +224,10 @@ public class ContainersLauncher extends AbstractService
break;
}
}
+
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index 64831e9..8da4ec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -23,10 +23,24 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
public interface ContainersMonitor extends Service,
EventHandler<ContainersMonitorEvent>, ResourceView {
- ResourceUtilization getContainersUtilization();
+
+ /**
+ * Get the aggregate resource utilization of containers running on the node,
+ * with a timestamp of the measurement.
+ * @param latest true if the latest result should be returned
+ * @return ResourceUtilization resource utilization of all containers
+ */
+ ContainersResourceUtilization getContainersUtilization(boolean latest);
+
+ /**
+ * Get the policy to over-allocate containers when over-allocation is on.
+ * @return null if over-allocation is turned off
+ */
+ NMAllocationPolicy getContainerOverAllocationPolicy();
float getVmemRatio();
@@ -66,4 +80,26 @@ public interface ContainersMonitor extends Service,
* containersMonitor.getVmemRatio());
resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores);
}
+
+ /**
+ * A snapshot of resource utilization of all containers with the timestamp.
+ */
+ final class ContainersResourceUtilization {
+ private final ResourceUtilization utilization;
+ private final long timestamp;
+
+ public ContainersResourceUtilization(
+ ResourceUtilization utilization, long timestamp) {
+ this.utilization = utilization;
+ this.timestamp = timestamp;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public ResourceUtilization getUtilization() {
+ return utilization;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 7873882..a045d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,8 +116,9 @@ public class ContainersMonitorImpl extends AbstractService implements
CPU, MEMORY
}
- private ResourceUtilization containersUtilization;
+ private ContainersResourceUtilization latestContainersUtilization;
+ private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
@@ -129,7 +134,8 @@ public class ContainersMonitorImpl extends AbstractService implements
this.monitoringThread = new MonitoringThread();
- this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ ResourceUtilization.newInstance(-1, -1, -1.0f), -1L);
}
@Override
@@ -365,6 +371,10 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
cpuPreemptionThreshold, memoryPreemptionThreshold);
+ // TODO make this configurable
+ this.overAllocationPolicy =
+ createOverAllocationPolicy(resourceThresholds);
+
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -372,6 +382,11 @@ public class ContainersMonitorImpl extends AbstractService implements
cpuPreemptionThreshold + ")");
}
+ protected NMAllocationPolicy createOverAllocationPolicy(
+ ResourceThresholds resourceThresholds) {
+ return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -655,7 +670,12 @@ public class ContainersMonitorImpl extends AbstractService implements
}
// Save the aggregated utilization of the containers
- setContainersUtilization(trackedContainersUtilization);
+ setLatestContainersUtilization(trackedContainersUtilization);
+
+ // check opportunity to start containers if over-allocation is on
+ if (context.isOverAllocationEnabled()) {
+ attemptToStartContainersUponLowUtilization();
+ }
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1045,12 +1065,34 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
- public ResourceUtilization getContainersUtilization() {
- return this.containersUtilization;
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ // TODO update containerUtilization if latest is true
+ return this.latestContainersUtilization;
+ }
+
+ @Override
+ public NMAllocationPolicy getContainerOverAllocationPolicy() {
+ return overAllocationPolicy;
+ }
+
+ private void setLatestContainersUtilization(ResourceUtilization utilization) {
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ utilization, System.currentTimeMillis());
}
- private void setContainersUtilization(ResourceUtilization utilization) {
- this.containersUtilization = utilization;
+ @VisibleForTesting
+ public void attemptToStartContainersUponLowUtilization() {
+ if (getContainerOverAllocationPolicy() != null) {
+ Resource available = getContainerOverAllocationPolicy()
+ .getAvailableResources();
+ if (available.getMemorySize() > 0 &&
+ available.getVirtualCores() > 0) {
+ eventDispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(null,
+ ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
new file mode 100644
index 0000000..86b3698
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the resource utilization tracker that equates
+ * resource utilization with the total resource allocated to the container.
+ */
+public class AllocationBasedResourceTracker
+ implements ResourceUtilizationTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private static final Resource UNAVAILABLE =
+ Resource.newInstance(0, 0);
+
+ private ResourceUtilization containersAllocation;
+ private ContainerScheduler scheduler;
+
+
+ AllocationBasedResourceTracker(ContainerScheduler scheduler) {
+ this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * Get the accumulation of totally allocated resources to containers.
+ * @return ResourceUtilization Resource Utilization.
+ */
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return this.containersAllocation;
+ }
+
+ /**
+ * Get the amount of resources that have not been allocated to containers.
+ * @return Resource resources that have not been allocated to containers.
+ */
+ protected Resource getUnallocatedResources() {
+ // unallocated resources = node capacity - containers allocation
+ // = -(container allocation - node capacity)
+ ResourceUtilization allocationClone =
+ ResourceUtilization.newInstance(containersAllocation);
+ getContainersMonitor()
+ .subtractNodeResourcesFromResourceUtilization(allocationClone);
+
+ Resource unallocated = UNAVAILABLE;
+ if (allocationClone.getCPU() <= 0 &&
+ allocationClone.getPhysicalMemory() <= 0 &&
+ allocationClone.getVirtualMemory() <= 0) {
+ int cpu = Math.round(allocationClone.getCPU() *
+ getContainersMonitor().getVCoresAllocatedForContainers());
+ long memory = allocationClone.getPhysicalMemory();
+ unallocated = Resource.newInstance(-memory, -cpu);
+ }
+ return unallocated;
+ }
+
+
+ @Override
+ public Resource getAvailableResources() {
+ return getUnallocatedResources();
+ }
+
+ /**
+ * Add Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerLaunched(Container container) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Subtract Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerReleased(Container container) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ public ContainersMonitor getContainersMonitor() {
+ return this.scheduler.getContainersMonitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 6e2b617..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An implementation of the {@link ResourceUtilizationTracker} that equates
- * resource utilization with the total resource allocated to the container.
- */
-public class AllocationBasedResourceUtilizationTracker implements
- ResourceUtilizationTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
-
- private ResourceUtilization containersAllocation;
- private ContainerScheduler scheduler;
-
- AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) {
- this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
- this.scheduler = scheduler;
- }
-
- /**
- * Get the accumulation of totally allocated resources to a container.
- * @return ResourceUtilization Resource Utilization.
- */
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return this.containersAllocation;
- }
-
- /**
- * Add Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void addContainerResources(Container container) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Subtract Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void subtractContainerResource(Container container) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Check if NM has resources available currently to run the container.
- * @param container Container.
- * @return True, if NM has resources available currently to run the container.
- */
- @Override
- public boolean hasResourcesAvailable(Container container) {
- long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
- return hasResourcesAvailable(pMemBytes,
- (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
- container.getResource().getVirtualCores());
- }
-
- private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
- int cpuVcores) {
- // Check physical memory.
- if (LOG.isDebugEnabled()) {
- LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
- this.containersAllocation.getPhysicalMemory(),
- (pMemBytes >> 20),
- (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
- }
- if (this.containersAllocation.getPhysicalMemory() +
- (int) (pMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getPmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before vMemCheck" +
- "[isEnabled={}, current={} + asked={} > allowed={}]",
- getContainersMonitor().isVmemCheckEnabled(),
- this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
- (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
- }
- // Check virtual memory.
- if (getContainersMonitor().isVmemCheckEnabled() &&
- this.containersAllocation.getVirtualMemory() +
- (int) (vMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getVmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before cpuCheck [asked={} > allowed={}]",
- this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers());
- }
- // Check CPU. Compare using integral values of cores to avoid decimal
- // inaccuracies.
- if (!hasEnoughCpu(this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers(), cpuVcores)) {
- return false;
- }
- return true;
- }
-
- /**
- * Returns whether there is enough space for coresRequested in totalCores.
- * Converts currentAllocation usage to nearest integer count before comparing,
- * as floats are inherently imprecise. NOTE: this calculation assumes that
- * requested core counts must be integers, and currentAllocation core count
- * must also be an integer.
- *
- * @param currentAllocation The current allocation, a float value from 0 to 1.
- * @param totalCores The total cores in the system.
- * @param coresRequested The number of cores requested.
- * @return True if currentAllocationtotalCores*coresRequested <=
- * totalCores.
- */
- public boolean hasEnoughCpu(float currentAllocation, long totalCores,
- int coresRequested) {
- // Must not cast here, as it would truncate the decimal digits.
- return Math.round(currentAllocation * totalCores)
- + coresRequested <= totalCores;
- }
-
- public ContainersMonitor getContainersMonitor() {
- return this.scheduler.getContainersMonitor();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index a61b9d1..0bebe44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +82,10 @@ public class ContainerScheduler extends AbstractService implements
// Queue of Guaranteed Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedGuaranteedContainers = new LinkedHashMap<>();
+ // sum of the resources requested by guaranteed containers in queue
+ private final Resource guaranteedResourcesDemanded =
+ Resource.newInstance(0, 0);
+
// Queue of Opportunistic Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedOpportunisticContainers = new LinkedHashMap<>();
@@ -88,6 +94,10 @@ public class ContainerScheduler extends AbstractService implements
// or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
+ // sum of the resources to be released by opportunistic containers that
+ // have been marked to be killed or paused.
+ private final Resource opportunisticResourcesToBeReleased =
+ Resource.newInstance(0, 0);
// Containers launched by the Scheduler will take a while to actually
// move to the RUNNING state, but should still be fair game for killing
@@ -128,6 +138,17 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
+ @VisibleForTesting
+ public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+ NodeManagerMetrics metrics, int qLength) {
+ super(ContainerScheduler.class.getName());
+ this.context = context;
+ this.dispatcher = dispatcher;
+ this.metrics = metrics;
+ this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
+ }
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -155,20 +176,16 @@ public class ContainerScheduler extends AbstractService implements
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
YarnConfiguration.
DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
+ // We assume over allocation configurations have been initialized
+ this.utilizationTracker = getResourceTracker();
}
- @VisibleForTesting
- public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
- NodeManagerMetrics metrics, int qLength) {
- super(ContainerScheduler.class.getName());
- this.context = context;
- this.dispatcher = dispatcher;
- this.metrics = metrics;
- this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
- this.utilizationTracker =
- new AllocationBasedResourceUtilizationTracker(this);
- this.opportunisticContainersStatus =
- OpportunisticContainersStatus.newInstance();
+ private AllocationBasedResourceTracker getResourceTracker() {
+ if (context.isOverAllocationEnabled()) {
+ return new UtilizationBasedResourceTracker(this);
+ } else {
+ return new AllocationBasedResourceTracker(this);
+ }
}
/**
@@ -191,14 +208,18 @@ public class ContainerScheduler extends AbstractService implements
if (event instanceof UpdateContainerSchedulerEvent) {
onUpdateContainer((UpdateContainerSchedulerEvent) event);
} else {
- LOG.error("Unknown event type on UpdateCOntainer: " + event.getType());
+ LOG.error("Unknown event type on UpdateContainer: " + event.getType());
}
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
case RECOVERY_COMPLETED:
- startPendingContainers(maxOppQueueLength <= 0);
+ startPendingContainers(false);
+ break;
+ case SCHEDULE_CONTAINERS:
+ startPendingContainers(true);
+ break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -213,10 +234,10 @@ public class ContainerScheduler extends AbstractService implements
ContainerId containerId = updateEvent.getContainer().getContainerId();
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
- this.utilizationTracker.subtractContainerResource(
+ this.utilizationTracker.containerReleased(
new ContainerImpl(getConfig(), null, null, null, null,
updateEvent.getOriginalToken(), context));
- this.utilizationTracker.addContainerResources(
+ this.utilizationTracker.containerLaunched(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
@@ -232,17 +253,20 @@ public class ContainerScheduler extends AbstractService implements
if (queuedOpportunisticContainers.remove(containerId) != null) {
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
- //Kill/pause opportunistic containers if any to make room for
- // promotion request
- reclaimOpportunisticContainerResources(updateEvent.getContainer());
+ Resources.addTo(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
+ startPendingContainers(true);
}
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
// containers
if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.subtractFrom(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
queuedOpportunisticContainers.put(containerId,
updateEvent.getContainer());
+ startPendingContainers(false);
}
}
try {
@@ -269,6 +293,7 @@ public class ContainerScheduler extends AbstractService implements
|| rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else if (execType == ExecutionType.OPPORTUNISTIC) {
queuedOpportunisticContainers
.put(container.getContainerId(), container);
@@ -279,7 +304,7 @@ public class ContainerScheduler extends AbstractService implements
}
} else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
- utilizationTracker.addContainerResources(container);
+ utilizationTracker.containerLaunched(container);
}
if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
&& rcs.getCapability() != null) {
@@ -344,65 +369,107 @@ public class ContainerScheduler extends AbstractService implements
}
private void onResourcesReclaimed(Container container) {
- oppContainersToKill.remove(container.getContainerId());
+ ContainerId containerId = container.getContainerId();
+
+ // This could be killed externally for eg. by the ContainerManager,
+ // in which case, the container might still be queued.
+ if (queuedOpportunisticContainers.remove(containerId) != null) {
+ return;
+ }
// This could be killed externally for eg. by the ContainerManager,
// in which case, the container might still be queued.
- Container queued =
- queuedOpportunisticContainers.remove(container.getContainerId());
- if (queued == null) {
- queuedGuaranteedContainers.remove(container.getContainerId());
+ if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
+ return;
+ }
+
+ if (oppContainersToKill.remove(containerId) != null) {
+ Resources.subtractFrom(
+ opportunisticResourcesToBeReleased, container.getResource());
}
// Requeue PAUSED containers
if (container.getContainerState() == ContainerState.PAUSED) {
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
- queuedGuaranteedContainers.put(container.getContainerId(), container);
+ queuedGuaranteedContainers.put(containerId, container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else {
- queuedOpportunisticContainers.put(
- container.getContainerId(), container);
+ queuedOpportunisticContainers.put(containerId, container);
}
}
// decrement only if it was a running container
- Container completedContainer = runningContainers.remove(container
- .getContainerId());
+ Container completedContainer = runningContainers.remove(containerId);
// only a running container releases resources upon completion
boolean resourceReleased = completedContainer != null;
if (resourceReleased) {
- this.utilizationTracker.subtractContainerResource(container);
+ this.utilizationTracker.containerReleased(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
+
+ // In case of over-allocation being turned on, we may need to reclaim
+ // more resources since the opportunistic containers that have been
+ // killed or paused may have not released as much resource as we need.
+ boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
+ startPendingContainers(reclaimOpportunisticResources);
}
}
/**
* Start pending containers in the queue.
- * @param forceStartGuaranteedContaieners When this is true, start guaranteed
- * container without looking at available resource
+ * @param reclaimOpportunisticResources if set to true, resources allocated
+ * to running OPPORTUNISTIC containers will be reclaimed in
+ * cases where there are GUARANTEED containers being queued
*/
- private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
- // Start guaranteed containers that are paused, if resources available.
- boolean resourcesAvailable = startContainers(
- queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
- // Start opportunistic containers, if resources available.
- if (resourcesAvailable) {
- startContainers(queuedOpportunisticContainers.values(), false);
+ private void startPendingContainers(boolean reclaimOpportunisticResources) {
+ // When opportunistic container not allowed (which is determined by
+ // max-queue length of pending opportunistic containers <= 0), start
+ // guaranteed containers without looking at available resources and
+ // skip scanning the queue of opportunistic containers
+ if (maxOppQueueLength <= 0) {
+ forcefullyStartGuaranteedContainers();
+ return;
+ }
+
+ Resource available = utilizationTracker.getAvailableResources();
+
+ // Start guaranteed containers that are queued, if resources available.
+ boolean allGuaranteedContainersLaunched =
+ startGuaranteedContainers(available);
+ // Start opportunistic containers, if resources available, which is true
+ // if all guaranteed containers in queue have been launched.
+ if (allGuaranteedContainersLaunched) {
+ startOpportunisticContainers(available);
+ } else {
+ // If not all guaranteed containers in queue are launched, we may need
+ // to reclaim resources from opportunistic containers that are running.
+ if (reclaimOpportunisticResources) {
+ reclaimOpportunisticContainerResources();
+ }
}
}
- private boolean startContainers(
- Collection<Container> containersToBeStarted, boolean force) {
- Iterator<Container> cIter = containersToBeStarted.iterator();
+ /**
+ * Try to launch as many GUARANTEED containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all queued GUARANTEED containers are launched
+ * or there is no GUARANTEED containers to launch
+ */
+ private boolean startGuaranteedContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (tryStartContainer(container, force)) {
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
} else {
resourcesAvailable = false;
}
@@ -410,25 +477,49 @@ public class ContainerScheduler extends AbstractService implements
return resourcesAvailable;
}
- private boolean tryStartContainer(Container container, boolean force) {
- boolean containerStarted = false;
- // call startContainer without checking available resource when force==true
- if (force || resourceAvailableToStartContainer(
- container)) {
+ /**
+ * Launch all queued GUARANTEED containers without checking resource
+ * availability. This is an optimization in cases where OPPORTUNISTIC
+ * containers are not allowed on the node.
+ */
+ private void forcefullyStartGuaranteedContainers() {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
+ while (cIter.hasNext()) {
+ Container container = cIter.next();
startContainer(container);
- containerStarted = true;
+ cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
}
- return containerStarted;
}
-
/**
- * Check if there is resource available to start a given container
- * immediately. (This can be extended to include overallocated resources)
- * @param container the container to start
- * @return true if container can be launched directly
+ * Try to launch as many OPPORTUNISTIC containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all OPPORTUNISTIC containers are launched
+ * or there is no OPPORTUNISTIC containers to launch
*/
- private boolean resourceAvailableToStartContainer(Container container) {
- return this.utilizationTracker.hasResourcesAvailable(container);
+ private boolean startOpportunisticContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedOpportunisticContainers.values().iterator();
+ boolean resourcesAvailable = true;
+ while (cIter.hasNext() && resourcesAvailable) {
+ Container container = cIter.next();
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
+ cIter.remove();
+ } else {
+ resourcesAvailable = false;
+ }
+ }
+ return resourcesAvailable;
+ }
+
+ private static boolean isResourceAvailable(
+ Resource resource, Container container) {
+ Resource left = Resources.subtract(resource, container.getResource());
+ return left.getMemorySize() >= 0 && left.getVirtualCores() >= 0;
}
private boolean enqueueContainer(Container container) {
@@ -438,6 +529,7 @@ public class ContainerScheduler extends AbstractService implements
boolean isQueued;
if (isGuaranteedContainer) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
isQueued = true;
} else {
if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
@@ -482,18 +574,7 @@ public class ContainerScheduler extends AbstractService implements
// enough number of opportunistic containers.
if (isGuaranteedContainer) {
enqueueContainer(container);
-
- // When opportunistic container not allowed (which is determined by
- // max-queue length of pending opportunistic containers <= 0), start
- // guaranteed containers without looking at available resources.
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
-
- // if the guaranteed container is queued, we need to preempt opportunistic
- // containers for make room for it
- if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
- reclaimOpportunisticContainerResources(container);
- }
+ startPendingContainers(true);
} else {
// Given an opportunistic container, we first try to start as many queuing
// guaranteed containers as possible followed by queuing opportunistic
@@ -511,19 +592,19 @@ public class ContainerScheduler extends AbstractService implements
}
@SuppressWarnings("unchecked")
- private void reclaimOpportunisticContainerResources(Container container) {
+ private void reclaimOpportunisticContainerResources() {
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources(
- container.getContainerId());
- // Kill the opportunistic containers that were chosen.
- for (Container contToReclaim : extraOppContainersToReclaim) {
+ pickOpportunisticContainersToReclaimResources();
+ killOpportunisticContainers(extraOppContainersToReclaim);
+ }
+
+ private void killOpportunisticContainers(
+ Collection<Container> containersToReclaim) {
+ for (Container contToReclaim : containersToReclaim) {
String preemptionAction = usePauseEventForPreemption == true ? "paused" :
- "resumed";
- LOG.info(
- "Container {} will be {} to start the "
- + "execution of guaranteed container {}.",
- contToReclaim.getContainerId(), preemptionAction,
- container.getContainerId());
+ "preempted";
+ LOG.info("Container {} will be {} to start the execution of guaranteed" +
+ " containers.", contToReclaim.getContainerId(), preemptionAction);
if (usePauseEventForPreemption) {
contToReclaim.sendPauseEvent(
@@ -534,6 +615,8 @@ public class ContainerScheduler extends AbstractService implements
"Container Killed to make room for Guaranteed Container.");
}
oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
+ Resources.addTo(
+ opportunisticResourcesToBeReleased, contToReclaim.getResource());
}
}
@@ -542,7 +625,7 @@ public class ContainerScheduler extends AbstractService implements
// Skip to put into runningContainers and addUtilization when recover
if (!runningContainers.containsKey(container.getContainerId())) {
runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.addContainerResources(container);
+ this.utilizationTracker.containerLaunched(container);
}
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -551,14 +634,12 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources(
- ContainerId containerToStartId) {
+ private List<Container> pickOpportunisticContainersToReclaimResources() {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp(
- containerToStartId);
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
@@ -577,15 +658,19 @@ public class ContainerScheduler extends AbstractService implements
continue;
}
extraOpportContainersToKill.add(runningCont);
+ // In the case of over-allocation, the running container may not
+ // release as much resources as it has requested, but we'll check
+ // again if more containers need to be killed/paused when this
+ // container is released.
ContainersMonitor.decreaseResourceUtilization(
getContainersMonitor(), resourcesToFreeUp,
runningCont.getResource());
}
}
if (!hasSufficientResources(resourcesToFreeUp)) {
- LOG.warn("There are no sufficient resources to start guaranteed [{}]" +
- "at the moment. Opportunistic containers are in the process of" +
- "being killed to make room.", containerToStartId);
+ LOG.warn("There are no sufficient resources to start guaranteed" +
+ " containers at the moment. Opportunistic containers are in" +
+ " the process of being killed to make room.");
}
return extraOpportContainersToKill;
}
@@ -600,34 +685,42 @@ public class ContainerScheduler extends AbstractService implements
* getContainersMonitor().getVCoresAllocatedForContainers()) <= 0;
}
- private ResourceUtilization resourcesToFreeUp(
- ContainerId containerToStartId) {
+ /**
+ * Determine how much resources are needed to be freed up to launch the given
+ * GUARANTEED container. Used to determine how many running OPPORTUNISTIC
+ * containers need to be killed/paused, assuming OPPORTUNISTIC containers to
+ * be killed/paused will release the amount of resources they have requested.
+ *
+ * If the node is over-allocating itself, this may cause not enough
+ * OPPORTUNISTIC containers being killed/paused in cases where the running
+ * OPPORTUNISTIC containers are not consuming fully their resource requests.
+ * We'd check again upon container completion events to see if more running
+ * OPPORTUNISTIC containers need to be killed/paused.
+ *
+ * @return the amount of resource needed to be reclaimed for this container
+ */
+ private ResourceUtilization resourcesToFreeUp() {
// Get allocation of currently allocated containers.
ResourceUtilization resourceAllocationToFreeUp = ResourceUtilization
- .newInstance(this.utilizationTracker.getCurrentUtilization());
-
- // Add to the allocation the allocation of the pending guaranteed
- // containers that will start before the current container will be started.
- for (Container container : queuedGuaranteedContainers.values()) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
- if (container.getContainerId().equals(containerToStartId)) {
- break;
- }
- }
+ .newInstance(0, 0, 0.0f);
+
+ // Add to the allocation the allocation of pending guaranteed containers.
+ ContainersMonitor.increaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, guaranteedResourcesDemanded);
// These resources are being freed, likely at the behest of another
// guaranteed container..
- for (Container container : oppContainersToKill.values()) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, opportunisticResourcesToBeReleased);
+
+ // Deduct any remaining resources available
+ Resource availableResources = utilizationTracker.getAvailableResources();
+ if (availableResources.getVirtualCores() > 0 &&
+ availableResources.getMemorySize() > 0) {
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, availableResources);
}
- // Subtract the overall node resources.
- getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
- resourceAllocationToFreeUp);
return resourceAllocationToFreeUp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 294eddf..9ad4f91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -28,5 +28,7 @@ public enum ContainerSchedulerEventType {
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
CONTAINER_PAUSED,
- RECOVERY_COMPLETED
+ RECOVERY_COMPLETED,
+ // Producer: Containers Monitor when over-allocation is on
+ SCHEDULE_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
new file mode 100644
index 0000000..58b73d2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * Keeps track of containers utilization over time and determines how much
+ * resources are available to launch containers when over-allocation is on.
+ */
+public abstract class NMAllocationPolicy {
+ protected final ResourceThresholds overAllocationThresholds;
+ protected final ContainersMonitor containersMonitor;
+
+ public NMAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ this.containersMonitor = containersMonitor;
+ this.overAllocationThresholds = overAllocationThresholds;
+ }
+
+ /**
+ * Handle container launch events.
+ * @param container the container that has been launched
+ */
+ public void containerLaunched(Container container) {
+
+ }
+
+ /**
+ * Handle container release events.
+ * @param container the container that has been released
+ */
+ public void containerReleased(Container container) {
+
+ }
+
+ /**
+ * Get the amount of resources to launch containers when
+ * over-allocation is turned on.
+ * @return the amount of resources available to launch containers
+ */
+ public abstract Resource getAvailableResources();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 3c17eca..98d99c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -38,22 +39,20 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
- * Add Container's resources to Node Utilization.
- * @param container Container.
+ * Get the amount of resources currently available to launch containers.
+ * @return Resource resources available to launch containers
*/
- void addContainerResources(Container container);
+ Resource getAvailableResources();
/**
- * Subtract Container's resources to Node Utilization.
+ * Add Container's resources to Node Utilization upon container launch.
* @param container Container.
*/
- void subtractContainerResource(Container container);
+ void containerLaunched(Container container);
/**
- * Check if NM has resources available currently to run the container.
+ * Subtract Container's resources to Node Utilization upon container release.
* @param container Container.
- * @return True, if NM has resources available currently to run the container.
*/
- boolean hasResourcesAvailable(Container container);
-
+ void containerReleased(Container container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
new file mode 100644
index 0000000..f486506
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * An implementation of NMAllocationPolicy based on the
+ * snapshot of the latest containers utilization to determine how much
+ * resources are available * to launch containers when over-allocation
+ * is turned on.
+ */
+public class SnapshotBasedOverAllocationPolicy
+ extends NMAllocationPolicy {
+
+ public SnapshotBasedOverAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ super(overAllocationThresholds, containersMonitor);
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ ResourceUtilization utilization =
+ containersMonitor.getContainersUtilization(true).getUtilization();
+ long memoryAvailable = Math.round(
+ overAllocationThresholds.getMemoryThreshold() *
+ containersMonitor.getPmemAllocatedForContainers()) -
+ (utilization.getPhysicalMemory() << 20);
+ int vcoreAvailable = Math.round(
+ (overAllocationThresholds.getCpuThreshold() - utilization.getCPU()) *
+ containersMonitor.getVCoresAllocatedForContainers());
+ return Resource.newInstance(memoryAvailable >> 20, vcoreAvailable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
new file mode 100644
index 0000000..6f9bc82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+* An resource availability tracker that determines if there are resources
+* available based on if there are unallocated resources or if there are
+* un-utilized resources.
+*/
+public class UtilizationBasedResourceTracker
+ extends AllocationBasedResourceTracker {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private final NMAllocationPolicy overAllocationPolicy;
+
+ UtilizationBasedResourceTracker(ContainerScheduler scheduler) {
+ super(scheduler);
+ this.overAllocationPolicy =
+ getContainersMonitor().getContainerOverAllocationPolicy();
+ }
+
+ @Override
+ public void containerLaunched(Container container) {
+ super.containerLaunched(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerLaunched(container);
+ }
+ }
+
+ @Override
+ public void containerReleased(Container container) {
+ super.containerReleased(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerReleased(container);
+ }
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ Resource resourceBasedOnAllocation = getUnallocatedResources();
+ Resource resourceBasedOnUtilization =
+ getResourcesAvailableBasedOnUtilization();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The amount of resources available based on allocation is " +
+ resourceBasedOnAllocation + ", based on utilization is " +
+ resourceBasedOnUtilization);
+ }
+
+ return Resources.componentwiseMax(resourceBasedOnAllocation,
+ resourceBasedOnUtilization);
+ }
+
+ /**
+ * Get the amount of resources based on the slack between
+ * the actual utilization and desired utilization.
+ * @return Resource resource available
+ */
+ private Resource getResourcesAvailableBasedOnUtilization() {
+ if (overAllocationPolicy == null) {
+ return Resources.none();
+ }
+
+ return overAllocationPolicy.getAvailableResources();
+ }
+
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return getContainersMonitor().getContainersUtilization(false)
+ .getUtilization();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 493aa4c..92613ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -345,6 +346,40 @@ public abstract class BaseContainerManagerTest {
fStates.contains(containerStatus.getState()));
}
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ ContainerSubState finalState)
+ throws InterruptedException, YarnException, IOException {
+ waitForContainerSubState(containerManager, containerID,
+ Arrays.asList(finalState), 20);
+ }
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ List<ContainerSubState> finalStates, int timeOutMax)
+ throws InterruptedException, YarnException, IOException {
+ List<ContainerId> list = new ArrayList<>();
+ list.add(containerID);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(list);
+ ContainerStatus containerStatus;
+ HashSet<ContainerSubState> fStates = new HashSet<>(finalStates);
+ int timeoutSecs = 0;
+ do {
+ Thread.sleep(1000);
+ containerStatus =
+ containerManager.getContainerStatuses(request)
+ .getContainerStatuses().get(0);
+ LOG.info("Waiting for container to get into one of states " + fStates
+ + ". Current state is " + containerStatus.getContainerSubState());
+ timeoutSecs += 1;
+ } while (!fStates.contains(containerStatus.getContainerSubState())
+ && timeoutSecs < timeOutMax);
+ LOG.info("Container state is " + containerStatus.getContainerSubState());
+ Assert.assertTrue("ContainerSubState is not correct (timedout)",
+ fStates.contains(containerStatus.getContainerSubState()));
+ }
+
+
public static void waitForApplicationState(
ContainerManagerImpl containerManager, ApplicationId appID,
ApplicationState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 8aee532..c071283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -288,7 +288,7 @@ public class TestContainersMonitorResourceChange {
// will be 0.
assertEquals(
"Resource utilization must be default with MonitorThread's first run",
- 0, containersMonitor.getContainersUtilization()
+ 0, containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
// Verify the container utilization value. Since atleast one round is done,
@@ -303,8 +303,9 @@ public class TestContainersMonitorResourceChange {
ContainersMonitorImpl containersMonitor, int timeoutMsecs)
throws InterruptedException {
int timeWaiting = 0;
- while (0 == containersMonitor.getContainersUtilization()
- .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) {
+ while (0 == containersMonitor.getContainersUtilization(false)
+ .getUtilization().compareTo(
+ ResourceUtilization.newInstance(0, 0, 0.0f))) {
if (timeWaiting >= timeoutMsecs) {
break;
}
@@ -316,7 +317,7 @@ public class TestContainersMonitorResourceChange {
}
assertTrue("Resource utilization is not changed from second run onwards",
- 0 != containersMonitor.getContainersUtilization()
+ 0 != containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
new file mode 100644
index 0000000..1e8bfdf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link AllocationBasedResourceTracker} class.
+ */
+public class TestAllocationBasedResourceTracker {
+
+ private ContainerScheduler mockContainerScheduler;
+
+ @Before
+ public void setup() {
+ mockContainerScheduler = mock(ContainerScheduler.class);
+ ContainersMonitor containersMonitor =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
+ conf.setInt(YarnConfiguration.NM_VCORES, 8);
+ containersMonitor.init(conf);
+ when(mockContainerScheduler.getContainersMonitor())
+ .thenReturn(containersMonitor);
+ }
+
+ /**
+ * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
+ * hasResourceAvailable should return false.
+ */
+ @Test
+ public void testHasResourcesAvailable() {
+ AllocationBasedResourceTracker tracker =
+ new AllocationBasedResourceTracker(mockContainerScheduler);
+ Container testContainer = mock(Container.class);
+ when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
+ for (int i = 0; i < 2; i++) {
+ Assert.assertTrue(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ tracker.containerLaunched(testContainer);
+ }
+ Assert.assertFalse(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ }
+
+ private static boolean isResourcesAvailable(
+ Resource available, Container container) {
+ return available.compareTo(container.getResource()) >= 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 82c2147..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for the {@link AllocationBasedResourceUtilizationTracker} class.
- */
-public class TestAllocationBasedResourceUtilizationTracker {
-
- private ContainerScheduler mockContainerScheduler;
-
- @Before
- public void setup() {
- mockContainerScheduler = mock(ContainerScheduler.class);
- ContainersMonitor containersMonitor =
- new ContainersMonitorImpl(mock(ContainerExecutor.class),
- mock(AsyncDispatcher.class), mock(Context.class));
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
- conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
- conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
- conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
- conf.setInt(YarnConfiguration.NM_VCORES, 8);
- containersMonitor.init(conf);
- when(mockContainerScheduler.getContainersMonitor())
- .thenReturn(containersMonitor);
- }
-
- /**
- * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
- * hasResourceAvailable should return false.
- */
- @Test
- public void testHasResourcesAvailable() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- Container testContainer = mock(Container.class);
- when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
- for (int i = 0; i < 2; i++) {
- Assert.assertTrue(tracker.hasResourcesAvailable(testContainer));
- tracker.addContainerResources(testContainer);
- }
- Assert.assertFalse(tracker.hasResourcesAvailable(testContainer));
- }
-
- /**
- * Test the case where the current allocation has been truncated to 0.8888891
- * (8/9 cores used). Request 1 additional core - hasEnoughCpu should return
- * true.
- */
- @Test
- public void testHasEnoughCpu() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- float currentAllocation = 0.8888891f;
- long totalCores = 9;
- int alreadyUsedCores = 8;
- Assert.assertTrue(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores));
- Assert.assertFalse(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores + 1));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-5473. Expose per-application over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9da8c64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9da8c64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9da8c64
Branch: refs/heads/YARN-1011
Commit: f9da8c64202842b692da2a05c623d48ce2603642
Parents: f00f7ad
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Jan 23 22:34:49 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:01:35 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/TypeConverter.java | 4 +-
.../hadoop/mapreduce/TestTypeConverter.java | 4 +-
.../hadoop/mapred/TestResourceMgrDelegate.java | 2 +-
.../records/ApplicationResourceUsageReport.java | 158 +++++++++++---
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 18 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 23 ++-
.../ApplicationResourceUsageReportPBImpl.java | 205 +++++++++++++++----
.../hadoop/yarn/util/resource/Resources.java | 20 ++
...pplicationHistoryManagerOnTimelineStore.java | 83 +++++---
.../TestApplicationHistoryClientService.java | 8 +-
...pplicationHistoryManagerOnTimelineStore.java | 18 +-
.../metrics/ApplicationMetricsConstants.java | 10 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 39 +++-
.../server/resourcemanager/RMAppManager.java | 18 +-
.../server/resourcemanager/RMServerUtils.java | 3 +-
.../metrics/TimelineServiceV1Publisher.java | 14 +-
.../metrics/TimelineServiceV2Publisher.java | 14 +-
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../records/ApplicationAttemptStateData.java | 144 +++++++++++--
.../pb/ApplicationAttemptStateDataPBImpl.java | 110 ++++++++--
.../server/resourcemanager/rmapp/RMAppImpl.java | 38 ++--
.../resourcemanager/rmapp/RMAppMetrics.java | 38 ++--
.../attempt/AggregateAppResourceUsage.java | 51 +++--
.../rmapp/attempt/RMAppAttemptImpl.java | 27 ++-
.../rmapp/attempt/RMAppAttemptMetrics.java | 49 +++--
.../ContainerResourceUsageReport.java | 46 +++++
.../rmcontainer/RMContainer.java | 4 +-
.../rmcontainer/RMContainerImpl.java | 117 ++++++++---
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 62 ++++--
.../scheduler/YarnScheduler.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 9 +-
.../webapp/FairSchedulerAppsBlock.java | 8 +
.../resourcemanager/webapp/RMAppBlock.java | 9 +-
.../resourcemanager/webapp/RMAppsBlock.java | 10 +
.../resourcemanager/webapp/dao/AppInfo.java | 78 +++++--
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/TestAppManager.java | 39 ++--
.../resourcemanager/TestApplicationACLs.java | 4 +-
.../resourcemanager/TestClientRMService.java | 45 ++--
.../TestContainerResourceUsage.java | 184 ++++++++++-------
.../applicationsmanager/MockAsm.java | 4 +-
.../TestCombinedSystemMetricsPublisher.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 23 ++-
.../TestSystemMetricsPublisherForV2.java | 12 +-
.../recovery/RMStateStoreTestBase.java | 12 +-
.../recovery/TestZKRMStateStore.java | 40 ++--
.../rmapp/TestRMAppTransitions.java | 6 +-
.../attempt/TestRMAppAttemptTransitions.java | 32 +--
.../capacity/TestCapacityScheduler.java | 4 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 20 +-
.../resourcemanager/webapp/TestAppPage.java | 2 +-
.../resourcemanager/webapp/TestRMWebApp.java | 3 +-
.../webapp/TestRMWebAppFairScheduler.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../router/webapp/RouterWebServiceUtil.java | 14 +-
.../router/webapp/TestRouterWebServiceUtil.java | 14 +-
61 files changed, 1431 insertions(+), 514 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index aea931a..23409fc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -527,8 +527,8 @@ public class TypeConverter {
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
(int)resourceUsageReport.getReservedResources().getMemorySize());
- jobStatus.setUsedMem(
- (int) resourceUsageReport.getUsedResources().getMemorySize());
+ jobStatus.setUsedMem((int) resourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
}
return jobStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
index 629a246..acd9c35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
@@ -94,7 +94,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
applicationReport.setApplicationResourceUsageReport(appUsageRpt);
JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
Assert.assertEquals(appStartTime, jobStatus.getStartTime());
@@ -133,7 +133,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt);
JobStatus status = TypeConverter.fromYarn(mockReport, jobFile);
Assert.assertNotNull("fromYarn returned null status", status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
index d6754f0..0506b27 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
@@ -137,7 +137,7 @@ public class TestResourceMgrDelegate {
Records.newRecord(Resource.class));
Mockito.when(appResources.getReservedResources()).thenReturn(
Records.newRecord(Resource.class));
- Mockito.when(appResources.getUsedResources()).thenReturn(
+ Mockito.when(appResources.getGuaranteedResourcesUsed()).thenReturn(
Records.newRecord(Resource.class));
Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
appResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index d2e33ff..4517db9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,21 +36,27 @@ public abstract class ApplicationResourceUsageReport {
@Private
@Unstable
public static ApplicationResourceUsageReport newInstance(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap, float queueUsagePerc,
- float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap, float queueUsagePerc,
+ float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
+
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setQueueUsagePercentage(queueUsagePerc);
report.setClusterUsagePercentage(clusterUsagePerc);
report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
@@ -87,16 +93,40 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNumReservedContainers(int num_reserved_containers);
/**
- * Get the used <code>Resource</code>. -1 for invalid/inaccessible reports.
- * @return the used <code>Resource</code>
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsedResources();
+ /**
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourcesUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourcesUsed(Resource resources);
+
+ /**
+ * Get the opportunistic <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the opportunistic <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourcesUsed();
+
@Private
@Unstable
- public abstract void setUsedResources(Resource resources);
+ public abstract void setOpportunisticResourcesUsed(Resource resources);
/**
* Get the reserved <code>Resource</code>. -1 for invalid/inaccessible reports.
@@ -123,42 +153,84 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNeededResources(Resource needed_resources);
/**
- * Set the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @param memory_seconds the aggregated amount of memory seconds
+ * Set the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @param memorySeconds the aggregated amount of guaranteed memory seconds
*/
@Private
@Unstable
- public abstract void setMemorySeconds(long memory_seconds);
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
/**
- * Get the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @return the aggregated amount of memory seconds
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ /**
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
/**
- * Set the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @param vcore_seconds the aggregated number of vcore seconds
+ * Set the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @param vcoreSeconds the aggregated number of guaranteed vcore seconds
*/
@Private
@Unstable
- public abstract void setVcoreSeconds(long vcore_seconds);
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @return the aggregated number of vcore seconds
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ /**
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
/**
+ * Get the aggregated amount of opportunistic memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of opportunistic memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the aggregated number of opportunistic vcores that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return the aggregated number of opportunistic vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the percentage of resources of the queue that the app is using.
* @return the percentage of resources of the queue that the app is using.
*/
@@ -231,23 +303,35 @@ public abstract class ApplicationResourceUsageReport {
public abstract long getPreemptedVcoreSeconds();
/**
- * Get the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
- * @return map containing the resource name and aggregated resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Private
@Unstable
- public abstract void setResourceSecondsMap(
+ public abstract void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
@@ -272,4 +356,24 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @param opportunisticResourceSecondsMap map containing the resource name
+ * and aggregated opportunistic resource-seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 49bf895..3c972b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -252,6 +252,8 @@ message ApplicationResourceUsageReportProto {
optional int64 preempted_vcore_seconds = 11;
repeated StringLongMapProto application_resource_usage_map = 12;
repeated StringLongMapProto application_preempted_resource_usage_map = 13;
+ optional ResourceProto used_opportunistic_resources = 14;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 15;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index a0e4e02..063c282 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -1144,15 +1144,20 @@ public class ApplicationCLI extends YarnCLI {
private void printResourceUsage(PrintWriter appReportStr,
ApplicationResourceUsageReport usageReport) {
- appReportStr.print("\tAggregate Resource Allocation : ");
+ appReportStr.print("\tAggregate Guaranteed Resource Allocation : ");
if (usageReport != null) {
- appReportStr.println(
- getResourceSecondsString(usageReport.getResourceSecondsMap()));
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getGuaranteedResourceSecondsMap()));
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getOpportunisticResourceSecondsMap()));
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println(getResourceSecondsString(
usageReport.getPreemptedResourceSecondsMap()));
} else {
appReportStr.println("N/A");
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println("N/A");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println("N/A");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index aed5258..27f276b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -170,13 +170,11 @@ public class TopCLI extends YarnCLI {
appReport.getApplicationResourceUsageReport()
.getNumReservedContainers();
displayStringsMap.put(Columns.RCONT, String.valueOf(reservedContainers));
- usedVirtualCores =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getVirtualCores();
+ usedVirtualCores = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getVirtualCores();
displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores));
- usedMemory =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getMemorySize() / 1024;
+ usedMemory = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getMemorySize() / 1024;
displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G");
reservedVirtualCores =
appReport.getApplicationResourceUsageReport().getReservedResources()
@@ -195,11 +193,11 @@ public class TopCLI extends YarnCLI {
progress = appReport.getProgress() * 100;
displayStringsMap.put(Columns.PROGRESS, String.format("%.2f", progress));
// store in GBSeconds
- memorySeconds =
- appReport.getApplicationResourceUsageReport().getMemorySeconds() / 1024;
+ memorySeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedMemorySeconds() / 1024;
displayStringsMap.put(Columns.MEMSECS, String.valueOf(memorySeconds));
- vcoreSeconds =
- appReport.getApplicationResourceUsageReport().getVcoreSeconds();
+ vcoreSeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedVcoreSeconds();
displayStringsMap.put(Columns.VCORESECS, String.valueOf(vcoreSeconds));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 6a24837..4778803 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -123,18 +123,27 @@ public class TestYarnCLI {
for (int i = 0; i < 2; ++i) {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
preemptedResoureSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), 2222L);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport
- .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(2, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
@@ -170,7 +179,9 @@ public class TestYarnCLI {
pw.println("\tTracking-URL : N/A");
pw.println("\tRPC Port : 124");
pw.println("\tAM Host : host");
- pw.println("\tAggregate Resource Allocation : " +
+ pw.println("\tAggregate Guaranteed Resource Allocation : " +
+ (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
+ pw.println("\tAggregate Opportunistic Resource Allocation : " +
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
pw.println("\tAggregate Resource Preempted : " +
(i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index 14ede5d..8ff1631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -40,12 +40,14 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto.Builder builder = null;
boolean viaProto = false;
- Resource usedResources;
- Resource reservedResources;
- Resource neededResources;
+ private Resource guaranteedResourceUsed;
+ private Resource reservedResources;
+ private Resource neededResources;
+ private Resource opportunisticResourceUsed;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationResourceUsageReportPBImpl() {
builder = ApplicationResourceUsageReportProto.newBuilder();
@@ -55,8 +57,9 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto proto) {
this.proto = proto;
viaProto = true;
- getResourceSecondsMap();
+ getGuaranteedResourceSecondsMap();
getPreemptedResourceSecondsMap();
+ getOpportunisticResourceSecondsMap();
}
public synchronized ApplicationResourceUsageReportProto getProto() {
@@ -87,8 +90,13 @@ extends ApplicationResourceUsageReport {
}
private void mergeLocalToBuilder() {
- if (this.usedResources != null) {
- builder.setUsedResources(convertToProtoFormat(this.usedResources));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setUsedResources(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setUsedOpportunisticResources(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.reservedResources != null) {
builder.setReservedResources(
@@ -99,19 +107,28 @@ extends ApplicationResourceUsageReport {
}
builder.clearApplicationResourceUsageMap();
builder.clearApplicationPreemptedResourceUsageMap();
+ builder.clearApplicationOpportunisticResourceUsageMap();
if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap
.isEmpty()) {
builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
- if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) {
+ if (guaranteedResourceSecondsMap != null &&
+ !guaranteedResourceSecondsMap.isEmpty()) {
builder.addAllApplicationResourceUsageMap(
- ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ guaranteedResourceSecondsMap));
+ }
+ if (opportunisticResourceSecondsMap != null &&
+ !opportunisticResourceSecondsMap.isEmpty()) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ opportunisticResourceSecondsMap));
}
- builder.setMemorySeconds(this.getMemorySeconds());
- builder.setVcoreSeconds(this.getVcoreSeconds());
+ builder.setMemorySeconds(this.getGuaranteedMemorySeconds());
+ builder.setVcoreSeconds(this.getGuaranteedVcoreSeconds());
builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds());
builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds());
}
@@ -157,24 +174,53 @@ extends ApplicationResourceUsageReport {
}
@Override
+ @Deprecated
public synchronized Resource getUsedResources() {
+ return getGuaranteedResourcesUsed();
+ }
+
+ @Override
+ public synchronized Resource getGuaranteedResourcesUsed() {
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- if (this.usedResources != null) {
- return this.usedResources;
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
}
if (!p.hasUsedResources()) {
return null;
}
- this.usedResources = convertFromProtoFormat(p.getUsedResources());
- return this.usedResources;
+ this.guaranteedResourceUsed = convertFromProtoFormat(p.getUsedResources());
+ return this.guaranteedResourceUsed;
}
@Override
- public synchronized void setUsedResources(Resource resources) {
+ public synchronized void setGuaranteedResourcesUsed(Resource resources) {
maybeInitBuilder();
if (resources == null)
builder.clearUsedResources();
- this.usedResources = resources;
+ this.guaranteedResourceUsed = resources;
+ }
+
+ @Override
+ public synchronized Resource getOpportunisticResourcesUsed() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
+ }
+ if (!p.hasUsedOpportunisticResources()) {
+ return null;
+ }
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getUsedOpportunisticResources());
+ return this.opportunisticResourceUsed;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourcesUsed(Resource resources) {
+ maybeInitBuilder();
+ if (resources == null) {
+ builder.clearUsedOpportunisticResources();
+ }
+ this.opportunisticResourceUsed = resources;
}
@Override
@@ -220,14 +266,14 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setMemorySeconds(long memory_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds);
+ public synchronized void setGuaranteedMemorySeconds(long memorySeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
}
@Override
- public synchronized long getMemorySeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ public synchronized long getGuaranteedMemorySeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
return tmp.get(ResourceInformation.MEMORY_MB.getName());
}
@@ -235,20 +281,50 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setVcoreSeconds(long vcore_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.VCORES.getName(), vcore_seconds);
+ @Deprecated
+ public synchronized long getMemorySeconds() {
+ return getGuaranteedMemorySeconds();
+ }
+
+ @Override
+ public synchronized void setGuaranteedVcoreSeconds(long vcoreSeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
}
@Override
+ public synchronized long getGuaranteedVcoreSeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ @Deprecated
public synchronized long getVcoreSeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ return getGuaranteedVcoreSeconds();
+ }
+
+ @Override
+ public synchronized long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public synchronized long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
return tmp.get(ResourceInformation.VCORES.getName());
}
return 0;
}
-
+
@Override
public synchronized void setPreemptedMemorySeconds(
long preemptedMemorySeconds) {
@@ -314,41 +390,47 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setResourceSecondsMap(
+ public synchronized void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap == null) {
return;
}
if (!resourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(0L);
+ this.setGuaranteedMemorySeconds(0L);
}
if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(0L);
+ this.setGuaranteedVcoreSeconds(0L);
}
}
@Override
+ @Deprecated
public synchronized Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ public synchronized Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils
+ this.guaranteedResourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
}
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(p.getVcoreSeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
}
- this.setMemorySeconds(p.getMemorySeconds());
- this.setVcoreSeconds(p.getVcoreSeconds());
- return this.resourceSecondsMap;
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
+ return this.guaranteedResourceSecondsMap;
}
@Override
@@ -389,4 +471,45 @@ extends ApplicationResourceUsageReport {
this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds());
return this.preemptedResourceSecondsMap;
}
+
+ @Override
+ public synchronized Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap = ProtoUtils
+ .convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ if (opportunisticResourceSecondsMap == null) {
+ return;
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 48c2c36..6c41015 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import java.util.Map;
+
/**
* Resources is a computation class which provides a set of apis to do
* mathematical operations on Resource object.
@@ -553,4 +555,22 @@ public class Resources {
Resource resource, Resource factor) {
return calculator.normalizeDown(resource, factor);
}
+
+ /**
+ * Merge resource usage entries from a map to another map.
+ * @param mergeFrom the map to merge from
+ * @param mergeTo the map to merge to
+ */
+ public static void mergeResourceSecondsMap(Map<String, Long> mergeFrom,
+ Map<String, Long> mergeTo) {
+ for (Map.Entry<String, Long> entry : mergeFrom.entrySet()) {
+ Long value = mergeTo.get(entry.getKey());
+ if (value != null) {
+ value += entry.getValue();
+ } else {
+ value = entry.getValue();
+ }
+ mergeTo.put(entry.getKey(), value);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 0b57717..85cc006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -330,29 +329,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
.toString();
}
- if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
- long vcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_METRICS);
- long memorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_METRICS);
- long preemptedMemorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
- long preemptedVcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- preemptedMemorySeconds);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
-
+ if (entityInfo.containsKey(
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)) {
+ Map<String, Long> guaranteedResourceSecondsMap =
+ extractGuaranteedResourceSecondsMap(entityInfo);
+ Map<String, Long> preemptedResoureSecondsMap =
+ extractPreemptedResourceSecondsMap(entityInfo);
+ Map<String, Long> opportunisticResourceSecondsMap =
+ extractOpportunisticResourceSecondsMap(entityInfo);
appResources = ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(0, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
@@ -456,6 +445,54 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static Map<String, Long> extractGuaranteedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+
+ long guaranteedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS);
+ long guaranteedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), guaranteedMemorySeconds);
+ guaranteedResourceSecondsMap.put(ResourceInformation.VCORES.getName(),
+ guaranteedVcoreSeconds);
+
+ return guaranteedResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractOpportunisticResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+
+ long opportunisticVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS);
+ long opportunisticMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(),
+ opportunisticMemorySeconds);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), opportunisticVcoreSeconds);
+
+ return opportunisticResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractPreemptedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
+
+ long preemptedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
+ long preemptedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
+ preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
+ preemptedMemorySeconds);
+ preemptedResoureSecondsMap
+ .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
+
+ return preemptedResoureSecondsMap;
+ }
private static long parseLong(Map<String, Object> entityInfo,
String infoKey) {
long result = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7ef6eca..6cb21ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -152,9 +152,13 @@ public class TestApplicationHistoryClientService {
ApplicationReport appReport = response.getApplicationReport();
Assert.assertNotNull(appReport);
Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
- .getMemorySeconds());
+ .getGuaranteedMemorySeconds());
Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
- .getVcoreSeconds());
+ .getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticMemorySeconds());
+ Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticVcoreSeconds());
Assert.assertEquals("application_0_0001", appReport.getApplicationId()
.toString());
Assert.assertEquals("test app type",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ecaaf1e..35996aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -250,9 +250,13 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport();
Assert.assertEquals(123,
- applicationResourceUsageReport.getMemorySeconds());
- Assert
- .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
+ applicationResourceUsageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123,
+ applicationResourceUsageReport.getOpportunisticMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getOpportunisticVcoreSeconds());
long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789;
if (i == 3) {
@@ -534,8 +538,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, 123);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, 345);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, 123);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, 345);
if (!missingPreemptMetrics) {
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 4cec409..b4123ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -70,12 +70,18 @@ public class ApplicationMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_APPLICATION_STATE";
- public static final String APP_CPU_METRICS =
+ public static final String APP_GUARANTEED_CPU_METRICS =
"YARN_APPLICATION_CPU";
- public static final String APP_MEM_METRICS =
+ public static final String APP_GUARANTEED_MEM_METRICS =
"YARN_APPLICATION_MEMORY";
+ public static final String APP_OPPORTUNISTIC_CPU_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_CPU";
+
+ public static final String APP_OPPORTUNISTIC_MEM_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_MEMORY";
+
public static final String APP_RESOURCE_PREEMPTED_CPU =
"YARN_APPLICATION_RESOURCE_PREEMPTED_CPU";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index a507e8d..c339b80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -456,19 +456,24 @@ public class BuilderUtils {
}
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index b4687a4..be6c17c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Times;
@@ -62,6 +63,8 @@ public class AppInfo {
protected int priority;
private long allocatedCpuVcores;
private long allocatedMemoryMB;
+ private long allocatedOpportunisticCpuVcores;
+ private long allocatedOpportunisticMemoryMB;
private long reservedCpuVcores;
private long reservedMemoryMB;
protected boolean unmanagedApplication;
@@ -100,15 +103,25 @@ public class AppInfo {
if (app.getApplicationResourceUsageReport() != null) {
runningContainers = app.getApplicationResourceUsageReport()
.getNumUsedContainers();
- if (app.getApplicationResourceUsageReport().getUsedResources() != null) {
- allocatedCpuVcores = app.getApplicationResourceUsageReport()
- .getUsedResources().getVirtualCores();
- allocatedMemoryMB = app.getApplicationResourceUsageReport()
- .getUsedResources().getMemorySize();
- reservedCpuVcores = app.getApplicationResourceUsageReport()
- .getReservedResources().getVirtualCores();
- reservedMemoryMB = app.getApplicationResourceUsageReport()
- .getReservedResources().getMemorySize();
+ Resource guaranteedResourceUsed =
+ app.getApplicationResourceUsageReport().getGuaranteedResourcesUsed();
+ if (guaranteedResourceUsed != null) {
+ allocatedCpuVcores = guaranteedResourceUsed.getVirtualCores();
+ allocatedMemoryMB = guaranteedResourceUsed.getMemorySize();
+ }
+ Resource opportunisticResourceUsed = app
+ .getApplicationResourceUsageReport().getOpportunisticResourcesUsed();
+ if (opportunisticResourceUsed != null) {
+ allocatedOpportunisticCpuVcores =
+ opportunisticResourceUsed.getVirtualCores();
+ allocatedOpportunisticMemoryMB =
+ opportunisticResourceUsed.getMemorySize();
+ }
+ Resource resourceReserved =
+ app.getApplicationResourceUsageReport().getReservedResources();
+ if (resourceReserved != null) {
+ reservedCpuVcores = resourceReserved.getVirtualCores();
+ reservedMemoryMB = resourceReserved.getMemorySize();
}
}
progress = app.getProgress() * 100; // in percent
@@ -168,6 +181,14 @@ public class AppInfo {
return allocatedMemoryMB;
}
+ public long getAllocatedOpportunisticCpuVcores() {
+ return allocatedOpportunisticCpuVcores;
+ }
+
+ public long getAllocatedOpportunisticMemoryMB() {
+ return allocatedOpportunisticMemoryMB;
+ }
+
public long getReservedCpuVcores() {
return reservedCpuVcores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 6ea55ed..b36d220 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -186,19 +186,27 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
- .add("memorySeconds", metrics.getMemorySeconds())
- .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("guaranteedMemorySeconds", metrics.getGuaranteedMemorySeconds())
+ .add("guaranteedVcoreSeconds", metrics.getGuaranteedVcoreSeconds())
+ .add("opportunisticMemorySeconds",
+ metrics.getOpportunisticMemorySeconds())
+ .add("opportunisticVcoreSeconds",
+ metrics.getOpportunisticVcoreSeconds())
.add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds())
.add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds())
.add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
.add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
.add("preemptedResources", metrics.getResourcePreempted())
.add("applicationType", app.getApplicationType())
- .add("resourceSeconds", StringHelper
- .getResourceSecondsString(metrics.getResourceSecondsMap()))
+ .add("guaranteedResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getGuaranteedResourceSecondsMap()))
.add("preemptedResourceSeconds", StringHelper
.getResourceSecondsString(
- metrics.getPreemptedResourceSecondsMap()));
+ metrics.getPreemptedResourceSecondsMap()))
+ .add("opportunisticResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getOpportunisticResourceSecondsMap()));
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index ab6bbcf..bf93df2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -481,7 +481,8 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
- Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>());
+ Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>(),
+ Resources.createResource(-1, -1), new HashMap<>());
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index dc5292b..909f7a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -142,10 +142,16 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
latestApplicationAttemptId);
}
RMAppMetrics appMetrics = app.getRMAppMetrics();
- entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
- appMetrics.getVcoreSeconds());
- entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
- appMetrics.getMemorySeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS,
+ appMetrics.getGuaranteedVcoreSeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS,
+ appMetrics.getGuaranteedMemorySeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ appMetrics.getOpportunisticVcoreSeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ appMetrics.getOpportunisticMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,
appMetrics.getPreemptedMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 89905e5..7a21e55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -193,11 +193,17 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_CPU_METRICS, timestamp,
- appMetrics.getVcoreSeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, timestamp,
+ appMetrics.getGuaranteedVcoreSeconds()));
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_MEM_METRICS, timestamp,
- appMetrics.getMemorySeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, timestamp,
+ appMetrics.getGuaranteedMemorySeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ timestamp, appMetrics.getOpportunisticVcoreSeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ timestamp, appMetrics.getOpportunisticMemorySeconds()));
entityMetrics.add(getTimelineMetric(
ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp,
appMetrics.getPreemptedMemorySeconds()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 242b5d0..4a7696f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -904,8 +904,9 @@ public abstract class RMStateStore extends AbstractService {
appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(),
credentials, appAttempt.getStartTime(),
- resUsage.getResourceUsageSecondsMap(),
- attempMetrics.getPreemptedResourceSecondsMap());
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ attempMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
getRMStateStoreEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9da8c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
index 2de071a..b08d2eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -45,8 +46,9 @@ public abstract class ApplicationAttemptStateData {
Credentials attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
- long finishTime, Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ long finishTime, Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@@ -59,19 +61,22 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.setAMContainerExitStatus(exitStatus);
attemptStateData.setFinishTime(finishTime);
- attemptStateData.setMemorySeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap,
+ attemptStateData.setGuaranteedMemorySeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
- attemptStateData.setVcoreSeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L));
+ attemptStateData.setGuaranteedVcoreSeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L));
attemptStateData.setPreemptedMemorySeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
attemptStateData.setPreemptedVcoreSeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.VCORES.getName(), 0L));
- attemptStateData.setResourceSecondsMap(resourceSecondsMap);
+ attemptStateData.setGuaranteedResourceSecondsMap(
+ guaranteedResourceSecondsMap);
+ attemptStateData.setOpportunisticResourceSecondsMap(
+ opportunisticResourcesSecondsMap);
attemptStateData
.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
return attemptStateData;
@@ -80,11 +85,13 @@ public abstract class ApplicationAttemptStateData {
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container masterContainer,
Credentials attemptTokens, long startTime,
- Map<String, Long> resourceSeondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
return newInstance(attemptId, masterContainer, attemptTokens, startTime,
null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
- resourceSeondsMap, preemptedResourceSecondsMap);
+ guaranteedResourceSecondsMap, preemptedResourceSecondsMap,
+ opportunisticResourcesSecondsMap);
}
@@ -180,30 +187,78 @@ public abstract class ApplicationAttemptStateData {
public abstract void setFinishTime(long finishTime);
/**
- * Get the <em>memory seconds</em> (in MB seconds) of the application.
- * @return <em>memory seconds</em> (in MB seconds) of the application
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setMemorySeconds(long memorySeconds);
/**
- * Get the <em>vcore seconds</em> of the application.
- * @return <em>vcore seconds</em> of the application
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setVcoreSeconds(long vcoreSeconds);
/**
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ @Private
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ @Private
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
+
+ /**
+ * Get the <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the <em>opportunistic vcore seconds</em> of the application.
+ * @return <em>opportunistic vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the <em>preempted memory seconds</em>
* (in MB seconds) of the application.
* @return <em>preempted memory seconds</em>
@@ -232,28 +287,54 @@ public abstract class ApplicationAttemptStateData {
public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of resources preempted that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
- * @return map containing the resource name and aggregated preempted
- * resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
+
+ /**
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * guaranteed resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
+
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
@@ -276,4 +357,27 @@ public abstract class ApplicationAttemptStateData {
@Unstable
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * opportunistic resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-6739. Crash NM at start time
if oversubscription is on but LinuxContainerExcutor or cgroup is off.
Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6739. Crash NM at start time if oversubscription is on but LinuxContainerExcutor or cgroup is off. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00678001
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00678001
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00678001
Branch: refs/heads/YARN-1011
Commit: 006780013df5624cce67b58dfdc20995a461878a
Parents: f9da8c6
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Apr 2 15:09:52 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:02:34 2018 -0700
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 25 ++++++++++++++++++++
1 file changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00678001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 70a8ebf..7873882 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -221,6 +222,7 @@ public class ContainersMonitorImpl extends AbstractService implements
initializeOverAllocation(conf);
if (context.isOverAllocationEnabled()) {
+ checkOverAllocationPrerequisites();
pmemCheckEnabled = true;
LOG.info("Force enabling physical memory checks because " +
"overallocation is enabled");
@@ -260,6 +262,29 @@ public class ContainersMonitorImpl extends AbstractService implements
super.serviceInit(this.conf);
}
+ /**
+ * Check all prerequisites for NM over-allocation.
+ */
+ private void checkOverAllocationPrerequisites() throws YarnException {
+ // LinuxContainerExecutor is required to enable overallocation
+ if (!(containerExecutor instanceof LinuxContainerExecutor)) {
+ throw new YarnException(LinuxContainerExecutor.class.getName() +
+ " is required for overallocation");
+ }
+ if (ResourceHandlerModule.getCGroupsHandler() == null) {
+ throw new YarnException("CGroups must be enabled to support" +
+ " overallocation");
+ }
+ if (ResourceHandlerModule.getCpuResourceHandler() == null) {
+ throw new YarnException(
+ "CGroups cpu isolation must be enabled to support overallocation");
+ }
+ if (ResourceHandlerModule.getMemoryResourceHandler() == null) {
+ throw new YarnException(
+ "CGroups memory isolation must be enabled for overallocation");
+ }
+ }
+
private boolean isContainerMonitorEnabled() {
return conf.getBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED,
YarnConfiguration.DEFAULT_NM_CONTAINER_MONITOR_ENABLED);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-7460. Exclude findbugs
warnings on SchedulerNode.numGuaranteedContainers and
numOpportunisticContainers. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7460. Exclude findbugs warnings on SchedulerNode.numGuaranteedContainers and numOpportunisticContainers. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33dcc79f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33dcc79f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33dcc79f
Branch: refs/heads/YARN-1011
Commit: 33dcc79f3f31effbf23a8e3a1f0394519595cbab
Parents: 495d43a
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Nov 20 13:27:32 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:03:25 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33dcc79f/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 216c3bd..ce3a3b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -71,6 +71,16 @@
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numOpportunisticContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numGuaranteedContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl$AppRejectedTransition" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDDS-460. Replication manager
failed to import container data. Contributed by Elek, Marton.
Posted by ha...@apache.org.
HDDS-460. Replication manager failed to import container data. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042bf74d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042bf74d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042bf74d
Branch: refs/heads/YARN-1011
Commit: 042bf74d5eb3dc627658b8c4c027628569169513
Parents: efdea85
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 23:51:50 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 23:51:50 2018 +0530
----------------------------------------------------------------------
.../statemachine/DatanodeStateMachine.java | 19 ++-
.../ReplicateContainerCommandHandler.java | 120 ++------------
.../replication/ContainerReplicator.java | 27 +++
.../DownloadAndImportReplicator.java | 136 ++++++++++++++++
.../replication/GrpcReplicationClient.java | 2 +-
.../replication/ReplicationSupervisor.java | 142 ++++++++++++++++
.../container/replication/ReplicationTask.java | 102 ++++++++++++
.../TestReplicateContainerCommandHandler.java | 163 -------------------
.../replication/TestReplicationSupervisor.java | 143 ++++++++++++++++
.../container/replication/package-info.java | 22 +++
10 files changed, 602 insertions(+), 274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 875d063..1bade8e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -40,7 +40,12 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.ReplicateContainerCommandHandler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.container.replication.ContainerReplicator;
+import org.apache.hadoop.ozone.container.replication.DownloadAndImportReplicator;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
@@ -69,6 +74,7 @@ public class DatanodeStateMachine implements Closeable {
private AtomicLong nextHB;
private Thread stateMachineThread = null;
private Thread cmdProcessThread = null;
+ private final ReplicationSupervisor supervisor;
/**
* Constructs a a datanode state machine.
@@ -89,14 +95,21 @@ public class DatanodeStateMachine implements Closeable {
new OzoneConfiguration(conf), context);
nextHB = new AtomicLong(Time.monotonicNow());
+ ContainerReplicator replicator =
+ new DownloadAndImportReplicator(container.getContainerSet(),
+ container.getDispatcher(),
+ new SimpleContainerDownloader(conf), new TarContainerPacker());
+
+ supervisor =
+ new ReplicationSupervisor(container.getContainerSet(), replicator, 10);
+
// When we add new handlers just adding a new handler here should do the
// trick.
commandDispatcher = CommandDispatcher.newBuilder()
.addHandler(new CloseContainerCommandHandler())
.addHandler(new DeleteBlocksCommandHandler(container.getContainerSet(),
conf))
- .addHandler(new ReplicateContainerCommandHandler(conf,
- container.getContainerSet(), container.getDispatcher()))
+ .addHandler(new ReplicateContainerCommandHandler(conf, supervisor))
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
@@ -295,6 +308,7 @@ public class DatanodeStateMachine implements Closeable {
public void startDaemon() {
Runnable startStateMachineTask = () -> {
try {
+ supervisor.start();
start();
LOG.info("Ozone container server started.");
} catch (Exception ex) {
@@ -323,6 +337,7 @@ public class DatanodeStateMachine implements Closeable {
*/
public synchronized void stopDaemon() {
try {
+ supervisor.stop();
context.setState(DatanodeStates.SHUTDOWN);
reportManager.shutdown();
this.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index cb677c2..09c379f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -16,33 +16,17 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-import java.io.FileInputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.container.replication.ContainerDownloader;
-import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -58,39 +42,19 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
static final Logger LOG =
LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
- private ContainerDispatcher containerDispatcher;
-
private int invocationCount;
private long totalTime;
- private ContainerDownloader downloader;
-
private Configuration conf;
- private TarContainerPacker packer = new TarContainerPacker();
-
- private ContainerSet containerSet;
-
- private Lock lock = new ReentrantLock();
+ private ReplicationSupervisor supervisor;
public ReplicateContainerCommandHandler(
Configuration conf,
- ContainerSet containerSet,
- ContainerDispatcher containerDispatcher,
- ContainerDownloader downloader) {
+ ReplicationSupervisor supervisor) {
this.conf = conf;
- this.containerSet = containerSet;
- this.downloader = downloader;
- this.containerDispatcher = containerDispatcher;
- }
-
- public ReplicateContainerCommandHandler(
- Configuration conf,
- ContainerSet containerSet,
- ContainerDispatcher containerDispatcher) {
- this(conf, containerSet, containerDispatcher,
- new SimpleContainerDownloader(conf));
+ this.supervisor = supervisor;
}
@Override
@@ -108,72 +72,12 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
String.format("Replication command is received for container %d "
+ "but the size of source datanodes was 0.", containerID));
- LOG.info("Starting replication of container {} from {}", containerID,
- sourceDatanodes);
- CompletableFuture<Path> tempTarFile = downloader
- .getContainerDataFromReplicas(containerID,
- sourceDatanodes);
-
- CompletableFuture<Void> result =
- tempTarFile.thenAccept(path -> {
- LOG.info("Container {} is downloaded, starting to import.",
- containerID);
- importContainer(containerID, path);
- });
-
- result.whenComplete((aVoid, throwable) -> {
- if (throwable != null) {
- LOG.error("Container replication was unsuccessful .", throwable);
- } else {
- LOG.info("Container {} is replicated successfully", containerID);
- }
- });
- } finally {
- updateCommandStatus(context, command, true, LOG);
+ ReplicationTask replicationTask =
+ new ReplicationTask(containerID, sourceDatanodes);
+ supervisor.addTask(replicationTask);
- }
- }
-
- protected void importContainer(long containerID, Path tarFilePath) {
- lock.lock();
- try {
- ContainerData originalContainerData;
- try (FileInputStream tempContainerTarStream = new FileInputStream(
- tarFilePath.toFile())) {
- byte[] containerDescriptorYaml =
- packer.unpackContainerDescriptor(tempContainerTarStream);
- originalContainerData = ContainerDataYaml.readContainer(
- containerDescriptorYaml);
- }
-
- try (FileInputStream tempContainerTarStream = new FileInputStream(
- tarFilePath.toFile())) {
-
- Handler handler = containerDispatcher.getHandler(
- originalContainerData.getContainerType());
-
- Container container = handler.importContainer(containerID,
- originalContainerData.getMaxSize(),
- tempContainerTarStream,
- packer);
-
- containerSet.addContainer(container);
- }
-
- } catch (Exception e) {
- LOG.error(
- "Can't import the downloaded container data id=" + containerID,
- e);
- try {
- Files.delete(tarFilePath);
- } catch (Exception ex) {
- LOG.error(
- "Container import is failed and the downloaded file can't be "
- + "deleted: "
- + tarFilePath.toAbsolutePath().toString());
- }
} finally {
- lock.unlock();
+ updateCommandStatus(context, command, true, LOG);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
new file mode 100644
index 0000000..827b9d6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Service to do the real replication task.
+ *
+ * An implementation should download the container and im
+ */
+public interface ContainerReplicator {
+ void replicate(ReplicationTask task);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
new file mode 100644
index 0000000..5ef5841
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Default replication implementation.
+ * <p>
+ * This class does the real job. Executes the download and import the container
+ * to the container set.
+ */
+public class DownloadAndImportReplicator implements ContainerReplicator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DownloadAndImportReplicator.class);
+
+ private final ContainerSet containerSet;
+
+ private final ContainerDispatcher containerDispatcher;
+
+ private final ContainerDownloader downloader;
+
+ private final TarContainerPacker packer;
+
+ public DownloadAndImportReplicator(
+ ContainerSet containerSet,
+ ContainerDispatcher containerDispatcher,
+ ContainerDownloader downloader,
+ TarContainerPacker packer) {
+ this.containerSet = containerSet;
+ this.containerDispatcher = containerDispatcher;
+ this.downloader = downloader;
+ this.packer = packer;
+ }
+
+ public void importContainer(long containerID, Path tarFilePath) {
+ try {
+ ContainerData originalContainerData;
+ try (FileInputStream tempContainerTarStream = new FileInputStream(
+ tarFilePath.toFile())) {
+ byte[] containerDescriptorYaml =
+ packer.unpackContainerDescriptor(tempContainerTarStream);
+ originalContainerData = ContainerDataYaml.readContainer(
+ containerDescriptorYaml);
+ }
+
+ try (FileInputStream tempContainerTarStream = new FileInputStream(
+ tarFilePath.toFile())) {
+
+ Handler handler = containerDispatcher.getHandler(
+ originalContainerData.getContainerType());
+
+ Container container = handler.importContainer(containerID,
+ originalContainerData.getMaxSize(),
+ tempContainerTarStream,
+ packer);
+
+ containerSet.addContainer(container);
+ }
+
+ } catch (Exception e) {
+ LOG.error(
+ "Can't import the downloaded container data id=" + containerID,
+ e);
+ try {
+ Files.delete(tarFilePath);
+ } catch (Exception ex) {
+ LOG.error(
+ "Container import is failed and the downloaded file can't be "
+ + "deleted: "
+ + tarFilePath.toAbsolutePath().toString());
+ }
+ }
+ }
+
+ @Override
+ public void replicate(ReplicationTask task) {
+ long containerID = task.getContainerId();
+
+ List<DatanodeDetails> sourceDatanodes = task.getSources();
+
+ LOG.info("Starting replication of container {} from {}", containerID,
+ sourceDatanodes);
+
+ CompletableFuture<Path> tempTarFile = downloader
+ .getContainerDataFromReplicas(containerID,
+ sourceDatanodes);
+
+ try {
+ //wait for the download. This thread pool is limiting the paralell
+ //downloads, so it's ok to block here and wait for the full download.
+ Path path = tempTarFile.get();
+ LOG.info("Container {} is downloaded, starting to import.",
+ containerID);
+ importContainer(containerID, path);
+ LOG.info("Container {} is replicated successfully", containerID);
+ task.setStatus(Status.DONE);
+ } catch (Exception e) {
+ LOG.error("Container replication was unsuccessful .", e);
+ task.setStatus(Status.FAILED);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 91d098f..3aafb0c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -157,8 +157,8 @@ public class GrpcReplicationClient {
public void onCompleted() {
try {
stream.close();
- response.complete(outputPath);
LOG.info("Container is downloaded to {}", outputPath);
+ response.complete(outputPath);
} catch (IOException e) {
response.completeExceptionally(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
new file mode 100644
index 0000000..1d8d5f6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Single point to schedule the downloading tasks based on priorities.
+ */
+public class ReplicationSupervisor {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReplicationSupervisor.class);
+
+ private final Set<Worker> threadPool = new HashSet<>();
+
+ private final Map<Long, ReplicationTask> queue = new TreeMap();
+
+ private final ContainerSet containerSet;
+
+ private final ContainerReplicator replicator;
+
+ private final int poolSize;
+
+ public ReplicationSupervisor(
+ ContainerSet containerSet,
+ ContainerReplicator replicator, int poolSize) {
+ this.containerSet = containerSet;
+ this.replicator = replicator;
+ this.poolSize = poolSize;
+ }
+
+ public synchronized void addTask(ReplicationTask task) {
+ queue.putIfAbsent(task.getContainerId(), task);
+ synchronized (threadPool) {
+ threadPool.notify();
+ }
+ }
+
+ public void start() {
+ for (int i = 0; i < poolSize; i++) {
+ Worker worker = new Worker();
+ Thread thread = new Thread(worker, "ContainerReplication-" + i);
+ thread.setDaemon(true);
+ thread.start();
+ threadPool.add(worker);
+ }
+ }
+
+ public synchronized ReplicationTask selectTask() {
+ for (ReplicationTask task : queue.values()) {
+ if (task.getStatus() == Status.QUEUED) {
+ if (containerSet.getContainer(task.getContainerId()) == null) {
+ task.setStatus(Status.DOWNLOADING);
+ return task;
+ } else {
+ LOG.debug("Container {} has already been downloaded.",
+ task.getContainerId());
+ queue.remove(task.getContainerId());
+ }
+ } else if (task.getStatus() == Status.FAILED) {
+ LOG.error(
+ "Container {} can't be downloaded from any of the datanodes.",
+ task.getContainerId());
+ queue.remove(task.getContainerId());
+ } else if (task.getStatus() == Status.DONE) {
+ queue.remove(task.getContainerId());
+ LOG.info("Container {} is replicated.", task.getContainerId());
+ }
+ }
+ //no available task.
+ return null;
+ }
+
+ public void stop() {
+ for (Worker worker : threadPool) {
+ worker.stop();
+ }
+ }
+
+ @VisibleForTesting
+ public int getQueueSize() {
+ return queue.size();
+ }
+
+ private class Worker implements Runnable {
+
+ private boolean running = true;
+
+ @Override
+ public void run() {
+ try {
+ while (running) {
+ ReplicationTask task = selectTask();
+ if (task == null) {
+ synchronized (threadPool) {
+ threadPool.wait();
+ }
+ } else {
+ replicator.replicate(task);
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("Error on doing replication", ex);
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ LOG.error("Error on waiting after failed replication task", e);
+ }
+ }
+ }
+
+ public void stop() {
+ running = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
new file mode 100644
index 0000000..9019811
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+/**
+ * The task to download a container from the sources.
+ */
+public class ReplicationTask {
+
+ private volatile Status status = Status.QUEUED;
+
+ private final long containerId;
+
+ private List<DatanodeDetails> sources;
+
+ private final Instant queued = Instant.now();
+
+ public ReplicationTask(long containerId,
+ List<DatanodeDetails> sources) {
+ this.containerId = containerId;
+ this.sources = sources;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReplicationTask that = (ReplicationTask) o;
+ return containerId == that.containerId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(containerId);
+ }
+
+ public long getContainerId() {
+ return containerId;
+ }
+
+ public List<DatanodeDetails> getSources() {
+ return sources;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(
+ Status status) {
+ this.status = status;
+ }
+
+ @Override
+ public String toString() {
+ return "ReplicationTask{" +
+ "status=" + status +
+ ", containerId=" + containerId +
+ ", sources=" + sources +
+ ", queued=" + queued +
+ '}';
+ }
+
+ public Instant getQueued() {
+ return queued;
+ }
+
+ /**
+ * Status of the replication.
+ */
+ public enum Status {
+ QUEUED,
+ DOWNLOADING,
+ FAILED,
+ DONE
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
deleted file mode 100644
index 6529922..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.replication.ContainerDownloader;
-import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.TestGenericTestUtils;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Test replication command handler.
- */
-public class TestReplicateContainerCommandHandler {
-
- private static final String EXCEPTION_MESSAGE = "Oh my god";
- private ReplicateContainerCommandHandler handler;
- private StubDownloader downloader;
- private ReplicateContainerCommand command;
- private List<Long> importedContainerIds;
-
- @Before
- public void init() {
- importedContainerIds = new ArrayList<>();
-
- OzoneConfiguration conf = new OzoneConfiguration();
- ContainerSet containerSet = Mockito.mock(ContainerSet.class);
- ContainerDispatcher containerDispatcher =
- Mockito.mock(ContainerDispatcher.class);
-
- downloader = new StubDownloader();
-
- handler = new ReplicateContainerCommandHandler(conf, containerSet,
- containerDispatcher, downloader) {
- @Override
- protected void importContainer(long containerID, Path tarFilePath) {
- importedContainerIds.add(containerID);
- }
- };
-
- //the command
- ArrayList<DatanodeDetails> datanodeDetails = new ArrayList<>();
- datanodeDetails.add(Mockito.mock(DatanodeDetails.class));
- datanodeDetails.add(Mockito.mock(DatanodeDetails.class));
-
- command = new ReplicateContainerCommand(1L, datanodeDetails);
- }
-
- @Test
- public void handle() throws TimeoutException, InterruptedException {
- //GIVEN
-
- //WHEN
- handler.handle(command, null, Mockito.mock(StateContext.class), null);
-
- TestGenericTestUtils
- .waitFor(() -> downloader.futureByContainers.size() == 1, 100, 2000);
-
- Assert.assertNotNull(downloader.futureByContainers.get(1L));
- downloader.futureByContainers.get(1L).complete(Paths.get("/tmp/test"));
-
- TestGenericTestUtils
- .waitFor(() -> importedContainerIds.size() == 1, 100, 2000);
- }
-
- @Test
- public void handleWithErrors() throws TimeoutException, InterruptedException {
- //GIVEN
- GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
- .captureLogs(ReplicateContainerCommandHandler.LOG);
-
- //WHEN
- handler.handle(command, null, Mockito.mock(StateContext.class), null);
-
- //THEN
- TestGenericTestUtils
- .waitFor(() -> downloader.futureByContainers.size() == 1, 100, 2000);
-
- Assert.assertNotNull(downloader.futureByContainers.get(1L));
- downloader.futureByContainers.get(1L)
- .completeExceptionally(new IllegalArgumentException(
- EXCEPTION_MESSAGE));
-
- TestGenericTestUtils
- .waitFor(() -> {
- String output = logCapturer.getOutput();
- return output.contains("unsuccessful") && output
- .contains(EXCEPTION_MESSAGE); },
- 100,
- 2000);
- }
-
- /**
- * Can't handle a command if there are no source replicas.
- */
- @Test(expected = IllegalArgumentException.class)
- public void handleWithoutReplicas()
- throws TimeoutException, InterruptedException {
- //GIVEN
- ReplicateContainerCommand commandWithoutReplicas =
- new ReplicateContainerCommand(1L, new ArrayList<>());
-
- //WHEN
- handler
- .handle(commandWithoutReplicas,
- null,
- Mockito.mock(StateContext.class),
- null);
-
- }
- private static class StubDownloader implements ContainerDownloader {
-
- private Map<Long, CompletableFuture<Path>> futureByContainers =
- new HashMap<>();
-
- @Override
- public void close() {
-
- }
-
- @Override
- public CompletableFuture<Path> getContainerDataFromReplicas(
- long containerId, List<DatanodeDetails> sources) {
- CompletableFuture<Path> future = new CompletableFuture<>();
- futureByContainers.put(containerId, future);
- return future;
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
new file mode 100644
index 0000000..d433319
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test the replication supervisor.
+ */
+public class TestReplicationSupervisor {
+
+ private OzoneConfiguration conf = new OzoneConfiguration();
+
+ @Test
+ public void normal() {
+ //GIVEN
+ ContainerSet set = new ContainerSet();
+
+ FakeReplicator replicator = new FakeReplicator(set);
+ ReplicationSupervisor supervisor =
+ new ReplicationSupervisor(set, replicator, 5);
+
+ List<DatanodeDetails> datanodes = IntStream.range(1, 3)
+ .mapToObj(v -> Mockito.mock(DatanodeDetails.class))
+ .collect(Collectors.toList());
+
+ try {
+ supervisor.start();
+ //WHEN
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(2L, datanodes));
+ supervisor.addTask(new ReplicationTask(2L, datanodes));
+ supervisor.addTask(new ReplicationTask(3L, datanodes));
+ try {
+ Thread.sleep(300);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ //THEN
+ System.out.println(replicator.replicated.get(0));
+
+ Assert
+ .assertEquals(3, replicator.replicated.size());
+
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @Test
+ public void duplicateMessageAfterAWhile() throws InterruptedException {
+ //GIVEN
+ ContainerSet set = new ContainerSet();
+
+ FakeReplicator replicator = new FakeReplicator(set);
+ ReplicationSupervisor supervisor =
+ new ReplicationSupervisor(set, replicator, 2);
+
+ List<DatanodeDetails> datanodes = IntStream.range(1, 3)
+ .mapToObj(v -> Mockito.mock(DatanodeDetails.class))
+ .collect(Collectors.toList());
+
+ try {
+ supervisor.start();
+ //WHEN
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ Thread.sleep(400);
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ Thread.sleep(300);
+
+ //THEN
+ System.out.println(replicator.replicated.get(0));
+
+ Assert
+ .assertEquals(1, replicator.replicated.size());
+
+ //the last item is still in the queue as we cleanup the queue during the
+ // selection
+ Assert.assertEquals(1, supervisor.getQueueSize());
+
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ private class FakeReplicator implements ContainerReplicator {
+
+ private List<ReplicationTask> replicated = new ArrayList<>();
+
+ private ContainerSet containerSet;
+
+ FakeReplicator(ContainerSet set) {
+ this.containerSet = set;
+ }
+
+ @Override
+ public void replicate(ReplicationTask task) {
+ KeyValueContainerData kvcd =
+ new KeyValueContainerData(task.getContainerId(), 100L);
+ KeyValueContainer kvc =
+ new KeyValueContainer(kvcd, conf);
+ try {
+ //download is slow
+ Thread.sleep(100);
+ replicated.add(task);
+ containerSet.addContainer(kvc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5c905e0
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Tests for the container replication.
+ */
+package org.apache.hadoop.ozone.container.replication;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDDS-394. Rename *Key Apis in
DatanodeContainerProtocol to *Block apis. Contributed Dinesh Chitlangia.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
new file mode 100644
index 0000000..9df4249
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.helpers;
+
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Tests to test block deleting service.
+ */
+public class TestBlockData {
+ static final Logger LOG = LoggerFactory.getLogger(TestBlockData.class);
+ @Rule
+ public TestRule timeout = new Timeout(10000);
+
+ static ContainerProtos.ChunkInfo buildChunkInfo(String name, long offset,
+ long len) {
+ return ContainerProtos.ChunkInfo.newBuilder()
+ .setChunkName(name).setOffset(offset).setLen(len).build();
+ }
+
+ @Test
+ public void testAddAndRemove() {
+ final BlockData computed = new BlockData(null);
+ final List<ContainerProtos.ChunkInfo> expected = new ArrayList<>();
+
+ assertChunks(expected, computed);
+ long offset = 0;
+ int n = 5;
+ for(int i = 0; i < n; i++) {
+ offset += assertAddChunk(expected, computed, offset);
+ }
+
+ for(; !expected.isEmpty();) {
+ removeChunk(expected, computed);
+ }
+ }
+
+ private static int chunkCount = 0;
+ static ContainerProtos.ChunkInfo addChunk(
+ List<ContainerProtos.ChunkInfo> expected, long offset) {
+ final long length = ThreadLocalRandom.current().nextLong(1000);
+ final ContainerProtos.ChunkInfo info =
+ buildChunkInfo("c" + ++chunkCount, offset, length);
+ expected.add(info);
+ return info;
+ }
+
+ static long assertAddChunk(List<ContainerProtos.ChunkInfo> expected,
+ BlockData computed, long offset) {
+ final ContainerProtos.ChunkInfo info = addChunk(expected, offset);
+ LOG.info("addChunk: " + toString(info));
+ computed.addChunk(info);
+ assertChunks(expected, computed);
+ return info.getLen();
+ }
+
+
+ static void removeChunk(List<ContainerProtos.ChunkInfo> expected,
+ BlockData computed) {
+ final int i = ThreadLocalRandom.current().nextInt(expected.size());
+ final ContainerProtos.ChunkInfo info = expected.remove(i);
+ LOG.info("removeChunk: " + toString(info));
+ computed.removeChunk(info);
+ assertChunks(expected, computed);
+ }
+
+ static void assertChunks(List<ContainerProtos.ChunkInfo> expected,
+ BlockData computed) {
+ final List<ContainerProtos.ChunkInfo> computedChunks = computed.getChunks();
+ Assert.assertEquals("expected=" + expected + "\ncomputed=" +
+ computedChunks, expected, computedChunks);
+ Assert.assertEquals(expected.stream().mapToLong(i -> i.getLen()).sum(),
+ computed.getSize());
+ }
+
+ static String toString(ContainerProtos.ChunkInfo info) {
+ return info.getChunkName() + ":" + info.getOffset() + "," + info.getLen();
+ }
+
+ static String toString(List<ContainerProtos.ChunkInfo> infos) {
+ return infos.stream().map(TestBlockData::toString)
+ .reduce((left, right) -> left + ", " + right)
+ .orElse("");
+ }
+
+ @Test
+ public void testSetChunks() {
+ final BlockData computed = new BlockData(null);
+ final List<ContainerProtos.ChunkInfo> expected = new ArrayList<>();
+
+ assertChunks(expected, computed);
+ long offset = 0;
+ int n = 5;
+ for(int i = 0; i < n; i++) {
+ offset += addChunk(expected, offset).getLen();
+ LOG.info("setChunk: " + toString(expected));
+ computed.setChunks(expected);
+ assertChunks(expected, computed);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestKeyData.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestKeyData.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestKeyData.java
deleted file mode 100644
index f57fe99..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestKeyData.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
-
-/**
- * Tests to test block deleting service.
- */
-public class TestKeyData {
- static final Logger LOG = LoggerFactory.getLogger(TestKeyData.class);
- @Rule
- public TestRule timeout = new Timeout(10000);
-
- static ContainerProtos.ChunkInfo buildChunkInfo(String name, long offset, long len) {
- return ContainerProtos.ChunkInfo.newBuilder()
- .setChunkName(name).setOffset(offset).setLen(len).build();
- }
-
- @Test
- public void testAddAndRemove() {
- final KeyData computed = new KeyData(null);
- final List<ContainerProtos.ChunkInfo> expected = new ArrayList<>();
-
- assertChunks(expected, computed);
- long offset = 0;
- int n = 5;
- for(int i = 0; i < n; i++) {
- offset += assertAddChunk(expected, computed, offset);
- }
-
- for(; !expected.isEmpty(); ) {
- removeChunk(expected, computed);
- }
- }
-
- private static int chunkCount = 0;
- static ContainerProtos.ChunkInfo addChunk(List<ContainerProtos.ChunkInfo> expected, long offset) {
- final long length = ThreadLocalRandom.current().nextLong(1000);
- final ContainerProtos.ChunkInfo info = buildChunkInfo("c" + ++chunkCount, offset, length);
- expected.add(info);
- return info;
- }
-
- static long assertAddChunk(List<ContainerProtos.ChunkInfo> expected, KeyData computed, long offset) {
- final ContainerProtos.ChunkInfo info = addChunk(expected, offset);
- LOG.info("addChunk: " + toString(info));
- computed.addChunk(info);
- assertChunks(expected, computed);
- return info.getLen();
- }
-
-
- static void removeChunk(List<ContainerProtos.ChunkInfo> expected, KeyData computed) {
- final int i = ThreadLocalRandom.current().nextInt(expected.size());
- final ContainerProtos.ChunkInfo info = expected.remove(i);
- LOG.info("removeChunk: " + toString(info));
- computed.removeChunk(info);
- assertChunks(expected, computed);
- }
-
- static void assertChunks(List<ContainerProtos.ChunkInfo> expected, KeyData computed) {
- final List<ContainerProtos.ChunkInfo> computedChunks = computed.getChunks();
- Assert.assertEquals("expected=" + expected + "\ncomputed=" + computedChunks, expected, computedChunks);
- Assert.assertEquals(expected.stream().mapToLong(i -> i.getLen()).sum(), computed.getSize());
- }
-
- static String toString(ContainerProtos.ChunkInfo info) {
- return info.getChunkName() + ":" + info.getOffset() + "," + info.getLen();
- }
-
- static String toString(List<ContainerProtos.ChunkInfo> infos) {
- return infos.stream().map(TestKeyData::toString)
- .reduce((left, right) -> left + ", " + right)
- .orElse("");
- }
-
- @Test
- public void testSetChunks() {
- final KeyData computed = new KeyData(null);
- final List<ContainerProtos.ChunkInfo> expected = new ArrayList<>();
-
- assertChunks(expected, computed);
- long offset = 0;
- int n = 5;
- for(int i = 0; i < n; i++) {
- offset += addChunk(expected, offset).getLen();
- LOG.info("setChunk: " + toString(expected));
- computed.setChunks(expected);
- assertChunks(expected, computed);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
index 73fa70d..78bf008 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java
@@ -25,12 +25,12 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
import org.junit.AfterClass;
@@ -71,14 +71,14 @@ public class TestCloseContainerHandler {
private final static String DATANODE_UUID = UUID.randomUUID().toString();
- private static final String baseDir = MiniDFSCluster.getBaseDirectory();
- private static final String volume1 = baseDir + "disk1";
- private static final String volume2 = baseDir + "disk2";
+ private static final String BASE_DIR = MiniDFSCluster.getBaseDirectory();
+ private static final String VOLUME_1 = BASE_DIR + "disk1";
+ private static final String VOLUME_2 = BASE_DIR + "disk2";
@BeforeClass
public static void setup() throws Exception {
conf = new Configuration();
- String dataDirKey = volume1 + "," + volume2;
+ String dataDirKey = VOLUME_1 + "," + VOLUME_2;
conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDirKey);
containerSet = new ContainerSet();
DatanodeDetails datanodeDetails =
@@ -160,31 +160,31 @@ public class TestCloseContainerHandler {
getTestBlockID(testContainerID);
Pipeline pipeline = createSingleNodePipeline();
List<ChunkInfo> chunkList = writeChunkBuilder(blockID, pipeline, 3);
- // the key should exist in the map
+ // the block should exist in the map
Assert.assertNotNull(
- openContainerBlockMap.getKeyDataMap(testContainerID)
+ openContainerBlockMap.getBlockDataMap(testContainerID)
.get(blockID.getLocalID()));
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunkProtoList = new LinkedList<>();
for (ChunkInfo i : chunkList) {
chunkProtoList.add(i.getProtoBufMessage());
}
- keyData.setChunks(chunkProtoList);
- ContainerProtos.PutKeyRequestProto.Builder putKeyRequestProto =
- ContainerProtos.PutKeyRequestProto.newBuilder();
- putKeyRequestProto.setKeyData(keyData.getProtoBufMessage());
+ blockData.setChunks(chunkProtoList);
+ ContainerProtos.PutBlockRequestProto.Builder putBlockRequestProto =
+ ContainerProtos.PutBlockRequestProto.newBuilder();
+ putBlockRequestProto.setBlockData(blockData.getProtoBufMessage());
ContainerProtos.ContainerCommandRequestProto.Builder request =
ContainerProtos.ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.PutKey);
+ request.setCmdType(ContainerProtos.Type.PutBlock);
request.setContainerID(blockID.getContainerID());
- request.setPutKey(putKeyRequestProto);
+ request.setPutBlock(putBlockRequestProto);
request.setTraceID(UUID.randomUUID().toString());
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
dispatcher.dispatch(request.build());
- //the open key should be removed from Map
+ //the open block should be removed from Map
Assert.assertNull(
- openContainerBlockMap.getKeyDataMap(testContainerID));
+ openContainerBlockMap.getBlockDataMap(testContainerID));
}
@Test
@@ -197,10 +197,10 @@ public class TestCloseContainerHandler {
List<ChunkInfo> chunkList = writeChunkBuilder(blockID, pipeline, 3);
// the key should exist in the map
Assert.assertNotNull(
- openContainerBlockMap.getKeyDataMap(testContainerID)
+ openContainerBlockMap.getBlockDataMap(testContainerID)
.get(blockID.getLocalID()));
Assert.assertTrue(
- openContainerBlockMap.getKeyDataMap(testContainerID)
+ openContainerBlockMap.getBlockDataMap(testContainerID)
.get(blockID.getLocalID()).getChunks().size() == 3);
ContainerProtos.DeleteChunkRequestProto.Builder deleteChunkProto =
ContainerProtos.DeleteChunkRequestProto.newBuilder();
@@ -220,7 +220,7 @@ public class TestCloseContainerHandler {
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
dispatcher.dispatch(request.build());
Assert.assertTrue(
- openContainerBlockMap.getKeyDataMap(testContainerID)
+ openContainerBlockMap.getBlockDataMap(testContainerID)
.get(blockID.getLocalID()).getChunks().size() == 2);
}
@@ -235,14 +235,14 @@ public class TestCloseContainerHandler {
List<ChunkInfo> chunkList = writeChunkBuilder(blockID, pipeline, 3);
Container container = containerSet.getContainer(testContainerID);
- KeyData keyData = openContainerBlockMap.
- getKeyDataMap(testContainerID).get(blockID.getLocalID());
+ BlockData blockData = openContainerBlockMap.
+ getBlockDataMap(testContainerID).get(blockID.getLocalID());
// the key should exist in the map
Assert.assertNotNull(
- openContainerBlockMap.getKeyDataMap(testContainerID)
+ openContainerBlockMap.getBlockDataMap(testContainerID)
.get(blockID.getLocalID()));
Assert.assertTrue(
- keyData.getChunks().size() == chunkList.size());
+ blockData.getChunks().size() == chunkList.size());
ContainerProtos.ContainerCommandRequestProto.Builder request =
ContainerProtos.ContainerCommandRequestProto.newBuilder();
request.setCmdType(ContainerProtos.Type.CloseContainer);
@@ -253,8 +253,9 @@ public class TestCloseContainerHandler {
request.setDatanodeUuid(pipeline.getLeader().getUuidString());
dispatcher.dispatch(request.build());
Assert.assertNull(
- openContainerBlockMap.getKeyDataMap(testContainerID));
+ openContainerBlockMap.getBlockDataMap(testContainerID));
// Make sure the key got committed
- Assert.assertNotNull(handler.getKeyManager().getKey(container, blockID));
+ Assert.assertNotNull(handler.getBlockManager()
+ .getBlock(container, blockID));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 3a36331..b6cdc9d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -29,20 +29,20 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerExcep
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
-import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
+import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.utils.MetadataStore;
import org.junit.After;
@@ -97,7 +97,7 @@ public class TestContainerPersistence {
private static ContainerSet containerSet;
private static VolumeSet volumeSet;
private static VolumeChoosingPolicy volumeChoosingPolicy;
- private static KeyManager keyManager;
+ private static BlockManager blockManager;
private static ChunkManager chunkManager;
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -126,7 +126,7 @@ public class TestContainerPersistence {
public void setupPaths() throws IOException {
containerSet = new ContainerSet();
volumeSet = new VolumeSet(DATANODE_UUID, conf);
- keyManager = new KeyManagerImpl(conf);
+ blockManager = new BlockManagerImpl(conf);
chunkManager = new ChunkManagerImpl();
for (String dir : conf.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
@@ -152,15 +152,15 @@ public class TestContainerPersistence {
return ContainerTestHelper.getTestContainerID();
}
- private Container addContainer(ContainerSet containerSet, long containerID)
+ private Container addContainer(ContainerSet cSet, long cID)
throws IOException {
- KeyValueContainerData data = new KeyValueContainerData(containerID,
+ KeyValueContainerData data = new KeyValueContainerData(cID,
ContainerTestHelper.CONTAINER_MAX_SIZE);
data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo");
KeyValueContainer container = new KeyValueContainer(data, conf);
container.create(volumeSet, volumeChoosingPolicy, SCM_ID);
- containerSet.addContainer(container);
+ cSet.addContainer(container);
return container;
}
@@ -184,7 +184,7 @@ public class TestContainerPersistence {
MetadataStore store = null;
try {
- store = KeyUtils.getDB(kvData, conf);
+ store = BlockUtils.getDB(kvData, conf);
Assert.assertNotNull(store);
} finally {
if (store != null) {
@@ -227,19 +227,19 @@ public class TestContainerPersistence {
Assert.assertFalse(containerSet.getContainerMap()
.containsKey(testContainerID1));
- // Adding key to a deleted container should fail.
+ // Adding block to a deleted container should fail.
exception.expect(StorageContainerException.class);
exception.expectMessage("Error opening DB.");
BlockID blockID1 = ContainerTestHelper.getTestBlockID(testContainerID1);
- KeyData someKey1 = new KeyData(blockID1);
+ BlockData someKey1 = new BlockData(blockID1);
someKey1.setChunks(new LinkedList<ContainerProtos.ChunkInfo>());
- keyManager.putKey(container1, someKey1);
+ blockManager.putBlock(container1, someKey1);
// Deleting a non-empty container should fail.
BlockID blockID2 = ContainerTestHelper.getTestBlockID(testContainerID2);
- KeyData someKey2 = new KeyData(blockID2);
+ BlockData someKey2 = new BlockData(blockID2);
someKey2.setChunks(new LinkedList<ContainerProtos.ChunkInfo>());
- keyManager.putKey(container2, someKey2);
+ blockManager.putBlock(container2, someKey2);
exception.expect(StorageContainerException.class);
exception.expectMessage(
@@ -325,7 +325,8 @@ public class TestContainerPersistence {
if (container == null) {
container = addContainer(containerSet, testContainerID);
}
- ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen);
+ ChunkInfo info = getChunk(
+ blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, data, COMBINED);
@@ -348,8 +349,8 @@ public class TestContainerPersistence {
}
/**
- * Writes many chunks of the same key into different chunk files and verifies
- * that we have that data in many files.
+ * Writes many chunks of the same block into different chunk files and
+ * verifies that we have that data in many files.
*
* @throws IOException
* @throws NoSuchAlgorithmException
@@ -425,7 +426,8 @@ public class TestContainerPersistence {
Container container = addContainer(containerSet, testContainerID);
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
- ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen);
+ ChunkInfo info = getChunk(
+ blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, data, COMBINED);
@@ -456,7 +458,8 @@ public class TestContainerPersistence {
Container container = addContainer(containerSet, testContainerID);
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
- ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen);
+ ChunkInfo info = getChunk(
+ blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, data, COMBINED);
@@ -500,7 +503,8 @@ public class TestContainerPersistence {
for (int x = 0; x < chunkCount; x++) {
// we are writing to the same chunk file but at different offsets.
long offset = x * datalen;
- ChunkInfo info = getChunk(blockID.getLocalID(), 0, offset, datalen);
+ ChunkInfo info = getChunk(
+ blockID.getLocalID(), 0, offset, datalen);
byte[] data = getData(datalen);
oldSha.update(data);
setDataChecksum(info, data);
@@ -531,7 +535,8 @@ public class TestContainerPersistence {
Container container = addContainer(containerSet, testContainerID);
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
- ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen);
+ ChunkInfo info = getChunk(
+ blockID.getLocalID(), 0, 0, datalen);
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, data, COMBINED);
@@ -542,37 +547,38 @@ public class TestContainerPersistence {
}
/**
- * Tests a put key and read key.
+ * Tests a put block and read block.
*
* @throws IOException
* @throws NoSuchAlgorithmException
*/
@Test
- public void testPutKey() throws IOException, NoSuchAlgorithmException {
+ public void testPutBlock() throws IOException, NoSuchAlgorithmException {
long testContainerID = getTestContainerID();
Container container = addContainer(containerSet, testContainerID);
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
ChunkInfo info = writeChunkHelper(blockID);
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
- keyManager.putKey(container, keyData);
- KeyData readKeyData = keyManager.getKey(container, keyData.getBlockID());
+ blockData.setChunks(chunkList);
+ blockManager.putBlock(container, blockData);
+ BlockData readBlockData = blockManager.
+ getBlock(container, blockData.getBlockID());
ChunkInfo readChunk =
- ChunkInfo.getFromProtoBuf(readKeyData.getChunks().get(0));
+ ChunkInfo.getFromProtoBuf(readBlockData.getChunks().get(0));
Assert.assertEquals(info.getChecksum(), readChunk.getChecksum());
}
/**
- * Tests a put key and read key.
+ * Tests a put block and read block.
*
* @throws IOException
* @throws NoSuchAlgorithmException
*/
@Test
- public void testPutKeyWithLotsOfChunks() throws IOException,
+ public void testPutBlockWithLotsOfChunks() throws IOException,
NoSuchAlgorithmException {
final int chunkCount = 2;
final int datalen = 1024;
@@ -603,66 +609,67 @@ public class TestContainerPersistence {
long writeCount = container.getContainerData().getWriteCount();
Assert.assertEquals(chunkCount, writeCount);
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunkProtoList = new LinkedList<>();
for (ChunkInfo i : chunkList) {
chunkProtoList.add(i.getProtoBufMessage());
}
- keyData.setChunks(chunkProtoList);
- keyManager.putKey(container, keyData);
- KeyData readKeyData = keyManager.getKey(container, keyData.getBlockID());
+ blockData.setChunks(chunkProtoList);
+ blockManager.putBlock(container, blockData);
+ BlockData readBlockData = blockManager.
+ getBlock(container, blockData.getBlockID());
ChunkInfo lastChunk = chunkList.get(chunkList.size() - 1);
ChunkInfo readChunk =
- ChunkInfo.getFromProtoBuf(readKeyData.getChunks().get(readKeyData
+ ChunkInfo.getFromProtoBuf(readBlockData.getChunks().get(readBlockData
.getChunks().size() - 1));
Assert.assertEquals(lastChunk.getChecksum(), readChunk.getChecksum());
}
/**
- * Deletes a key and tries to read it back.
+ * Deletes a block and tries to read it back.
*
* @throws IOException
* @throws NoSuchAlgorithmException
*/
@Test
- public void testDeleteKey() throws IOException, NoSuchAlgorithmException {
+ public void testDeleteBlock() throws IOException, NoSuchAlgorithmException {
long testContainerID = getTestContainerID();
Container container = addContainer(containerSet, testContainerID);
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
ChunkInfo info = writeChunkHelper(blockID);
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
- keyManager.putKey(container, keyData);
- keyManager.deleteKey(container, blockID);
+ blockData.setChunks(chunkList);
+ blockManager.putBlock(container, blockData);
+ blockManager.deleteBlock(container, blockID);
exception.expect(StorageContainerException.class);
- exception.expectMessage("Unable to find the key.");
- keyManager.getKey(container, keyData.getBlockID());
+ exception.expectMessage("Unable to find the block.");
+ blockManager.getBlock(container, blockData.getBlockID());
}
/**
- * Tries to Deletes a key twice.
+ * Tries to Deletes a block twice.
*
* @throws IOException
* @throws NoSuchAlgorithmException
*/
@Test
- public void testDeleteKeyTwice() throws IOException,
+ public void testDeleteBlockTwice() throws IOException,
NoSuchAlgorithmException {
long testContainerID = getTestContainerID();
Container container = addContainer(containerSet, testContainerID);
BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID);
ChunkInfo info = writeChunkHelper(blockID);
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
- keyManager.putKey(container, keyData);
- keyManager.deleteKey(container, blockID);
+ blockData.setChunks(chunkList);
+ blockManager.putBlock(container, blockData);
+ blockManager.deleteBlock(container, blockID);
exception.expect(StorageContainerException.class);
- exception.expectMessage("Unable to find the key.");
- keyManager.deleteKey(container, blockID);
+ exception.expectMessage("Unable to find the block.");
+ blockManager.deleteBlock(container, blockID);
}
/**
@@ -722,8 +729,9 @@ public class TestContainerPersistence {
try {
container.update(newMetadata, false);
} catch (StorageContainerException ex) {
- Assert.assertEquals("Updating a closed container without force option " +
- "is not allowed. ContainerID: " + testContainerID, ex.getMessage());
+ Assert.assertEquals("Updating a closed container without " +
+ "force option is not allowed. ContainerID: " +
+ testContainerID, ex.getMessage());
}
// Update with force flag, it should be success.
@@ -741,53 +749,55 @@ public class TestContainerPersistence {
}
- private KeyData writeKeyHelper(BlockID blockID)
+ private BlockData writeBlockHelper(BlockID blockID)
throws IOException, NoSuchAlgorithmException {
ChunkInfo info = writeChunkHelper(blockID);
- KeyData keyData = new KeyData(blockID);
+ BlockData blockData = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>();
chunkList.add(info.getProtoBufMessage());
- keyData.setChunks(chunkList);
- return keyData;
+ blockData.setChunks(chunkList);
+ return blockData;
}
@Test
- public void testListKey() throws Exception {
+ public void testListBlock() throws Exception {
long testContainerID = getTestContainerID();
Container container = addContainer(containerSet, testContainerID);
- List<BlockID> expectedKeys = new ArrayList<>();
+ List<BlockID> expectedBlocks = new ArrayList<>();
for (int i = 0; i < 10; i++) {
BlockID blockID = new BlockID(testContainerID, i);
- expectedKeys.add(blockID);
- KeyData kd = writeKeyHelper(blockID);
- keyManager.putKey(container, kd);
+ expectedBlocks.add(blockID);
+ BlockData kd = writeBlockHelper(blockID);
+ blockManager.putBlock(container, kd);
}
- // List all keys
- List<KeyData> result = keyManager.listKey(container, 0, 100);
+ // List all blocks
+ List<BlockData> result = blockManager.listBlock(
+ container, 0, 100);
Assert.assertEquals(10, result.size());
int index = 0;
for (int i = index; i < result.size(); i++) {
- KeyData data = result.get(i);
+ BlockData data = result.get(i);
Assert.assertEquals(testContainerID, data.getContainerID());
- Assert.assertEquals(expectedKeys.get(i).getLocalID(), data.getLocalID());
+ Assert.assertEquals(expectedBlocks.get(i).getLocalID(),
+ data.getLocalID());
index++;
}
- // List key with startKey filter
- long k6 = expectedKeys.get(6).getLocalID();
- result = keyManager.listKey(container, k6, 100);
+ // List block with startBlock filter
+ long k6 = expectedBlocks.get(6).getLocalID();
+ result = blockManager.listBlock(container, k6, 100);
Assert.assertEquals(4, result.size());
for (int i = 6; i < 10; i++) {
- Assert.assertEquals(expectedKeys.get(i).getLocalID(),
+ Assert.assertEquals(expectedBlocks.get(i).getLocalID(),
result.get(i - 6).getLocalID());
}
// Count must be >0
exception.expect(IllegalArgumentException.class);
exception.expectMessage("Count must be a positive number.");
- keyManager.listKey(container, 0, -1);
+ blockManager.listBlock(container, 0, -1);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 3c77687..a129ed0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
@@ -69,6 +69,9 @@ import static org.apache.hadoop.hdds
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+/**
+ * Tests for Block deletion.
+ */
public class TestBlockDeletion {
private static OzoneConfiguration conf = null;
private static ObjectStore store;
@@ -229,7 +232,7 @@ public class TestBlockDeletion {
throws IOException {
return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
try {
- MetadataStore db = KeyUtils.getDB((KeyValueContainerData)
+ MetadataStore db = BlockUtils.getDB((KeyValueContainerData)
dnContainerSet.getContainer(blockID.getContainerID())
.getContainerData(), conf);
Assert.assertNotNull(db.get(Longs.toByteArray(blockID.getLocalID())));
@@ -244,7 +247,7 @@ public class TestBlockDeletion {
throws IOException {
return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
try {
- MetadataStore db = KeyUtils.getDB((KeyValueContainerData)
+ MetadataStore db = BlockUtils.getDB((KeyValueContainerData)
dnContainerSet.getContainer(blockID.getContainerID())
.getContainerData(), conf);
Assert.assertNull(db.get(Longs.toByteArray(blockID.getLocalID())));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index 5dd88fb..a3c92fb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -68,7 +68,8 @@ public class TestOzoneContainer {
conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath());
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
- conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
container = new OzoneContainer(TestUtils.randomDatanodeDetails(),
conf, null);
@@ -129,7 +130,7 @@ public class TestOzoneContainer {
static void runTestOzoneContainerViaDataNode(
long testContainerID, XceiverClientSpi client) throws Exception {
ContainerProtos.ContainerCommandRequestProto
- request, writeChunkRequest, putKeyRequest,
+ request, writeChunkRequest, putBlockRequest,
updateRequest1, updateRequest2;
ContainerProtos.ContainerCommandResponseProto response,
updateResponse1, updateResponse2;
@@ -138,46 +139,50 @@ public class TestOzoneContainer {
Pipeline pipeline = client.getPipeline();
createContainerForTesting(client, testContainerID);
- writeChunkRequest = writeChunkForContainer(client, testContainerID, 1024);
+ writeChunkRequest = writeChunkForContainer(client, testContainerID,
+ 1024);
// Read Chunk
- request = ContainerTestHelper.getReadChunkRequest(pipeline, writeChunkRequest
- .getWriteChunk());
+ request = ContainerTestHelper.getReadChunkRequest(
+ pipeline, writeChunkRequest.getWriteChunk());
response = client.sendCommand(request);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
- // Put Key
- putKeyRequest = ContainerTestHelper.getPutKeyRequest(pipeline, writeChunkRequest
- .getWriteChunk());
+ // Put Block
+ putBlockRequest = ContainerTestHelper.getPutBlockRequest(
+ pipeline, writeChunkRequest.getWriteChunk());
- response = client.sendCommand(putKeyRequest);
+ response = client.sendCommand(putBlockRequest);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
- Assert
- .assertTrue(putKeyRequest.getTraceID().equals(response.getTraceID()));
+ Assert.assertTrue(putBlockRequest.getTraceID()
+ .equals(response.getTraceID()));
- // Get Key
- request = ContainerTestHelper.getKeyRequest(pipeline, putKeyRequest.getPutKey());
+ // Get Block
+ request = ContainerTestHelper.
+ getBlockRequest(pipeline, putBlockRequest.getPutBlock());
response = client.sendCommand(request);
- int chunksCount = putKeyRequest.getPutKey().getKeyData().getChunksCount();
- ContainerTestHelper.verifyGetKey(request, response, chunksCount);
+ int chunksCount = putBlockRequest.getPutBlock().getBlockData().
+ getChunksCount();
+ ContainerTestHelper.verifyGetBlock(request, response, chunksCount);
- // Delete Key
+ // Delete Block
request =
- ContainerTestHelper.getDeleteKeyRequest(pipeline, putKeyRequest.getPutKey());
+ ContainerTestHelper.getDeleteBlockRequest(
+ pipeline, putBlockRequest.getPutBlock());
response = client.sendCommand(request);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
//Delete Chunk
- request = ContainerTestHelper.getDeleteChunkRequest(pipeline, writeChunkRequest
- .getWriteChunk());
+ request = ContainerTestHelper.getDeleteChunkRequest(
+ pipeline, writeChunkRequest.getWriteChunk());
response = client.sendCommand(request);
Assert.assertNotNull(response);
@@ -249,7 +254,7 @@ public class TestOzoneContainer {
final ContainerProtos.ContainerCommandRequestProto getSmallFileRequest
= ContainerTestHelper.getReadSmallFileRequest(client.getPipeline(),
- smallFileRequest.getPutSmallFile().getKey());
+ smallFileRequest.getPutSmallFile().getBlock());
response = client.sendCommand(getSmallFileRequest);
Assert.assertArrayEquals(
smallFileRequest.getPutSmallFile().getData().toByteArray(),
@@ -269,7 +274,7 @@ public class TestOzoneContainer {
XceiverClientGrpc client = null;
ContainerProtos.ContainerCommandResponseProto response;
ContainerProtos.ContainerCommandRequestProto
- writeChunkRequest, putKeyRequest, request;
+ writeChunkRequest, putBlockRequest, request;
try {
OzoneConfiguration conf = newOzoneConfiguration();
@@ -283,18 +288,19 @@ public class TestOzoneContainer {
long containerID = ContainerTestHelper.getTestContainerID();
createContainerForTesting(client, containerID);
- writeChunkRequest = writeChunkForContainer(client, containerID, 1024);
+ writeChunkRequest = writeChunkForContainer(client, containerID,
+ 1024);
- putKeyRequest = ContainerTestHelper.getPutKeyRequest(client.getPipeline(),
- writeChunkRequest.getWriteChunk());
- // Put key before closing.
- response = client.sendCommand(putKeyRequest);
+ putBlockRequest = ContainerTestHelper.getPutBlockRequest(
+ client.getPipeline(), writeChunkRequest.getWriteChunk());
+ // Put block before closing.
+ response = client.sendCommand(putBlockRequest);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.SUCCESS,
response.getResult());
Assert.assertTrue(
- putKeyRequest.getTraceID().equals(response.getTraceID()));
+ putBlockRequest.getTraceID().equals(response.getTraceID()));
// Close the contianer.
request = ContainerTestHelper.getCloseContainer(
@@ -325,25 +331,26 @@ public class TestOzoneContainer {
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
- // Put key will fail on a closed container.
- response = client.sendCommand(putKeyRequest);
+ // Put block will fail on a closed container.
+ response = client.sendCommand(putBlockRequest);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.CLOSED_CONTAINER_IO,
response.getResult());
- Assert
- .assertTrue(putKeyRequest.getTraceID().equals(response.getTraceID()));
+ Assert.assertTrue(putBlockRequest.getTraceID()
+ .equals(response.getTraceID()));
- // Get key must work on the closed container.
- request = ContainerTestHelper.getKeyRequest(client.getPipeline(),
- putKeyRequest.getPutKey());
+ // Get block must work on the closed container.
+ request = ContainerTestHelper.getBlockRequest(client.getPipeline(),
+ putBlockRequest.getPutBlock());
response = client.sendCommand(request);
- int chunksCount = putKeyRequest.getPutKey().getKeyData().getChunksCount();
- ContainerTestHelper.verifyGetKey(request, response, chunksCount);
+ int chunksCount = putBlockRequest.getPutBlock().getBlockData()
+ .getChunksCount();
+ ContainerTestHelper.verifyGetBlock(request, response, chunksCount);
- // Delete Key must fail on a closed container.
+ // Delete block must fail on a closed container.
request =
- ContainerTestHelper.getDeleteKeyRequest(client.getPipeline(),
- putKeyRequest.getPutKey());
+ ContainerTestHelper.getDeleteBlockRequest(client.getPipeline(),
+ putBlockRequest.getPutBlock());
response = client.sendCommand(request);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.CLOSED_CONTAINER_IO,
@@ -365,7 +372,7 @@ public class TestOzoneContainer {
XceiverClientGrpc client = null;
ContainerProtos.ContainerCommandResponseProto response;
ContainerProtos.ContainerCommandRequestProto request,
- writeChunkRequest, putKeyRequest;
+ writeChunkRequest, putBlockRequest;
try {
OzoneConfiguration conf = newOzoneConfiguration();
@@ -378,17 +385,18 @@ public class TestOzoneContainer {
long containerID = ContainerTestHelper.getTestContainerID();
createContainerForTesting(client, containerID);
- writeChunkRequest = writeChunkForContainer(client, containerID, 1024);
+ writeChunkRequest = writeChunkForContainer(
+ client, containerID, 1024);
- putKeyRequest = ContainerTestHelper.getPutKeyRequest(client.getPipeline(),
- writeChunkRequest.getWriteChunk());
+ putBlockRequest = ContainerTestHelper.getPutBlockRequest(
+ client.getPipeline(), writeChunkRequest.getWriteChunk());
// Put key before deleting.
- response = client.sendCommand(putKeyRequest);
+ response = client.sendCommand(putBlockRequest);
Assert.assertNotNull(response);
Assert.assertEquals(ContainerProtos.Result.SUCCESS,
response.getResult());
Assert.assertTrue(
- putKeyRequest.getTraceID().equals(response.getTraceID()));
+ putBlockRequest.getTraceID().equals(response.getTraceID()));
// Container cannot be deleted forcibly because
// the container is not closed.
@@ -529,7 +537,7 @@ public class TestOzoneContainer {
writeChunkForContainer(XceiverClientSpi client,
long containerID, int dataLen) throws Exception {
// Write Chunk
- BlockID blockID = ContainerTestHelper.getTestBlockID(containerID);;
+ BlockID blockID = ContainerTestHelper.getTestBlockID(containerID);
ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
ContainerTestHelper.getWriteChunkRequest(client.getPipeline(),
blockID, dataLen);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
index 8c83fd3..c875a7e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerStateMachine.java
@@ -89,7 +89,7 @@ public class TestContainerStateMachine {
// add putKey request
ContainerCommandRequestProto putKeyProto = ContainerTestHelper
- .getPutKeyRequest(pipeline, writeChunkProto.getWriteChunk());
+ .getPutBlockRequest(pipeline, writeChunkProto.getWriteChunk());
RaftClientRequest putKeyRequest = getRaftClientRequest(putKeyProto);
TransactionContext createContainerCtxt =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
index f309715..7144005 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
@@ -1209,8 +1209,8 @@ public class TestOzoneManager {
//Disabling this test
@Ignore("Disabling this test until Open Key is fixed.")
public void testExpiredOpenKey() throws Exception {
-// BackgroundService openKeyCleanUpService = ((KeyManagerImpl)cluster
-// .getOzoneManager().getKeyManager()).getOpenKeyCleanupService();
+// BackgroundService openKeyCleanUpService = ((BlockManagerImpl)cluster
+// .getOzoneManager().getBlockManager()).getOpenKeyCleanupService();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index a2d95e8..84a4028 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@ -103,7 +103,7 @@ public class TestContainerSmallFile {
}
@Test
- public void testInvalidKeyRead() throws Exception {
+ public void testInvalidBlockRead() throws Exception {
String traceID = UUID.randomUUID().toString();
ContainerWithPipeline container =
storageContainerLocationClient.allocateContainer(
@@ -116,7 +116,7 @@ public class TestContainerSmallFile {
container.getContainerInfo().getContainerID(), traceID);
thrown.expect(StorageContainerException.class);
- thrown.expectMessage("Unable to find the key");
+ thrown.expectMessage("Unable to find the block");
BlockID blockID = ContainerTestHelper.getTestBlockID(
container.getContainerInfo().getContainerID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
index f82b0d3..08e7808 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
@@ -107,7 +107,7 @@ public class TestGetCommittedBlockLengthAndPutKey {
// Now, explicitly make a putKey request for the block.
ContainerProtos.ContainerCommandRequestProto putKeyRequest =
ContainerTestHelper
- .getPutKeyRequest(pipeline, writeChunkRequest.getWriteChunk());
+ .getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
client.sendCommand(putKeyRequest);
response = ContainerProtocolCalls
.getCommittedBlockLength(client, blockID, traceID);
@@ -155,7 +155,7 @@ public class TestGetCommittedBlockLengthAndPutKey {
}
@Test
- public void tesGetCommittedBlockLengthForInvalidBlock() throws Exception {
+ public void testGetCommittedBlockLengthForInvalidBlock() throws Exception {
String traceID = UUID.randomUUID().toString();
ContainerWithPipeline container = storageContainerLocationClient
.allocateContainer(xceiverClientManager.getType(),
@@ -174,7 +174,7 @@ public class TestGetCommittedBlockLengthAndPutKey {
ContainerProtocolCalls.getCommittedBlockLength(client, blockID, traceID);
Assert.fail("Expected exception not thrown");
} catch (StorageContainerException sce) {
- Assert.assertTrue(sce.getMessage().contains("Unable to find the key"));
+ Assert.assertTrue(sce.getMessage().contains("Unable to find the block"));
}
xceiverClientManager.releaseClient(client);
}
@@ -216,7 +216,7 @@ public class TestGetCommittedBlockLengthAndPutKey {
@Test
public void tesPutKeyResposne() throws Exception {
- ContainerProtos.PutKeyResponseProto response;
+ ContainerProtos.PutBlockResponseProto response;
String traceID = UUID.randomUUID().toString();
ContainerWithPipeline container = storageContainerLocationClient
.allocateContainer(xceiverClientManager.getType(),
@@ -239,8 +239,8 @@ public class TestGetCommittedBlockLengthAndPutKey {
// Now, explicitly make a putKey request for the block.
ContainerProtos.ContainerCommandRequestProto putKeyRequest =
ContainerTestHelper
- .getPutKeyRequest(pipeline, writeChunkRequest.getWriteChunk());
- response = client.sendCommand(putKeyRequest).getPutKey();
+ .getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
+ response = client.sendCommand(putKeyRequest).getPutBlock();
// make sure the block ids in the request and response are same.
// This will also ensure that closing the container committed the block
// on the Datanodes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 02cd985..7eb2ec2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
@@ -278,29 +278,29 @@ public class TestKeys {
}
static void runTestPutKey(PutHelper helper) throws Exception {
- final ClientProtocol client = helper.client;
+ final ClientProtocol helperClient = helper.client;
helper.putKey();
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
- List<OzoneKey> keyList = client
+ List<OzoneKey> keyList = helperClient
.listKeys(helper.getVol().getName(), helper.getBucket().getName(), null,
null, 10);
Assert.assertEquals(1, keyList.size());
// test list key using a more efficient call
String newkeyName = OzoneUtils.getRequestID().toLowerCase();
- OzoneOutputStream ozoneOutputStream = client
+ OzoneOutputStream ozoneOutputStream = helperClient
.createKey(helper.getVol().getName(), helper.getBucket().getName(),
newkeyName, 0, replicationType, replicationFactor);
ozoneOutputStream.close();
- keyList = client
+ keyList = helperClient
.listKeys(helper.getVol().getName(), helper.getBucket().getName(), null,
null, 10);
Assert.assertEquals(2, keyList.size());
// test new put key with invalid volume/bucket name
try {
- ozoneOutputStream = client
+ ozoneOutputStream = helperClient
.createKey("invalid-volume", helper.getBucket().getName(), newkeyName,
0, replicationType, replicationFactor);
ozoneOutputStream.close();
@@ -312,7 +312,7 @@ public class TestKeys {
}
try {
- ozoneOutputStream = client
+ ozoneOutputStream = helperClient
.createKey(helper.getVol().getName(), "invalid-bucket", newkeyName, 0,
replicationType, replicationFactor);
ozoneOutputStream.close();
@@ -380,7 +380,7 @@ public class TestKeys {
}
static void runTestPutAndGetKey(PutHelper helper) throws Exception {
- final ClientProtocol client = helper.client;
+ final ClientProtocol helperClient = helper.client;
String keyName = helper.putKey();
assertNotNull(helper.getBucket());
@@ -427,7 +427,8 @@ public class TestKeys {
// test new get key with invalid volume/bucket name
try {
- client.getKey("invalid-volume", helper.getBucket().getName(), keyName);
+ helperClient.getKey(
+ "invalid-volume", helper.getBucket().getName(), keyName);
fail("Get key should have thrown " + "when using invalid volume name.");
} catch (IOException e) {
GenericTestUtils
@@ -435,7 +436,8 @@ public class TestKeys {
}
try {
- client.getKey(helper.getVol().getName(), "invalid-bucket", keyName);
+ helperClient.getKey(
+ helper.getVol().getName(), "invalid-bucket", keyName);
fail("Get key should have thrown " + "when using invalid bucket name.");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains(
@@ -476,7 +478,7 @@ public class TestKeys {
}
static void runTestPutAndListKey(PutHelper helper) throws Exception {
- ClientProtocol client = helper.client;
+ ClientProtocol helperClient = helper.client;
helper.putKey();
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
@@ -495,7 +497,7 @@ public class TestKeys {
List<OzoneKey> keyList1 =
IteratorUtils.toList(helper.getBucket().listKeys(null, null));
// test list key using a more efficient call
- List<OzoneKey> keyList2 = client
+ List<OzoneKey> keyList2 = helperClient
.listKeys(helper.getVol().getName(), helper.getBucket().getName(), null,
null, 100);
@@ -515,7 +517,7 @@ public class TestKeys {
}
// test maxLength parameter of list keys
- keyList2 = client
+ keyList2 = helperClient
.listKeys(helper.getVol().getName(), helper.getBucket().getName(), null,
null, 1);
Assert.assertEquals(1, keyList2.size());
@@ -523,7 +525,7 @@ public class TestKeys {
// test startKey parameter of list keys
keyList1 = IteratorUtils
.toList(helper.getBucket().listKeys("list-key", "list-key4"));
- keyList2 = client
+ keyList2 = helperClient
.listKeys(helper.getVol().getName(), helper.getBucket().getName(),
"list-key", "list-key4", 100);
Assert.assertEquals(5, keyList1.size());
@@ -532,7 +534,7 @@ public class TestKeys {
// test prefix parameter of list keys
keyList1 =
IteratorUtils.toList(helper.getBucket().listKeys("list-key2", null));
- keyList2 = client
+ keyList2 = helperClient
.listKeys(helper.getVol().getName(), helper.getBucket().getName(),
"list-key2", null, 100);
Assert.assertTrue(
@@ -542,7 +544,7 @@ public class TestKeys {
// test new list keys with invalid volume/bucket name
try {
- client.listKeys("invalid-volume", helper.getBucket().getName(),
+ helperClient.listKeys("invalid-volume", helper.getBucket().getName(),
null, null, 100);
fail("List keys should have thrown when using invalid volume name.");
} catch (IOException e) {
@@ -551,7 +553,7 @@ public class TestKeys {
}
try {
- client.listKeys(helper.getVol().getName(), "invalid-bucket", null,
+ helperClient.listKeys(helper.getVol().getName(), "invalid-bucket", null,
null, 100);
fail("List keys should have thrown when using invalid bucket name.");
} catch (IOException e) {
@@ -697,10 +699,10 @@ public class TestKeys {
.KeyValueContainer);
KeyValueContainer container = (KeyValueContainer) cm.getContainerSet()
.getContainer(location.getBlockID().getContainerID());
- KeyData blockInfo = keyValueHandler
- .getKeyManager().getKey(container, location.getBlockID());
- KeyValueContainerData containerData = (KeyValueContainerData) container
- .getContainerData();
+ BlockData blockInfo = keyValueHandler
+ .getBlockManager().getBlock(container, location.getBlockID());
+ KeyValueContainerData containerData =
+ (KeyValueContainerData) container.getContainerData();
File dataDir = new File(containerData.getChunksPath());
for (ContainerProtos.ChunkInfo chunkInfo : blockInfo.getChunks()) {
File chunkFile = dataDir.toPath()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
index 9c451e2..3e740d9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
@@ -54,7 +54,7 @@ public class BucketManagerImpl implements BucketManager {
/**
* MetadataDB is maintained in MetadataManager and shared between
- * BucketManager and VolumeManager. (and also by KeyManager)
+ * BucketManager and VolumeManager. (and also by BlockManager)
*
* BucketManager uses MetadataDB to store bucket level information.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
index 2da60de..eb533e8 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java
@@ -102,7 +102,7 @@ public class ScmBlockLocationTestIngClient implements ScmBlockLocationProtocol {
}
/**
- * Returns Fake blocks to the KeyManager so we get blocks in the Database.
+ * Returns Fake blocks to the BlockManager so we get blocks in the Database.
* @param size - size of the block.
* @param type Replication Type
* @param factor - Replication factor
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 5ac7e0a..8811d91 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -48,17 +48,16 @@ import java.util.Random;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
-
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .PutBlockRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .GetBlockRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ReadChunkRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.WriteChunkRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .PutKeyRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .GetKeyRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
@@ -141,7 +140,7 @@ public class BenchMarkDatanodeDispatcher {
long containerID = containers.get(y);
BlockID blockID = new BlockID(containerID, key);
dispatcher
- .dispatch(getPutKeyCommand(blockID, chunkName));
+ .dispatch(getPutBlockCommand(blockID, chunkName));
dispatcher.dispatch(getWriteChunkCommand(blockID, chunkName));
}
}
@@ -213,38 +212,39 @@ public class BenchMarkDatanodeDispatcher {
return builder.build();
}
- private ContainerCommandRequestProto getPutKeyCommand(
+ private ContainerCommandRequestProto getPutBlockCommand(
BlockID blockID, String chunkKey) {
- PutKeyRequestProto.Builder putKeyRequest = PutKeyRequestProto
+ PutBlockRequestProto.Builder putBlockRequest = PutBlockRequestProto
.newBuilder()
- .setKeyData(getKeyData(blockID, chunkKey));
+ .setBlockData(getBlockData(blockID, chunkKey));
ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
.newBuilder();
- request.setCmdType(ContainerProtos.Type.PutKey)
+ request.setCmdType(ContainerProtos.Type.PutBlock)
.setContainerID(blockID.getContainerID())
.setTraceID(getBlockTraceID(blockID))
.setDatanodeUuid(datanodeUuid)
- .setPutKey(putKeyRequest);
+ .setPutBlock(putBlockRequest);
return request.build();
}
- private ContainerCommandRequestProto getGetKeyCommand(BlockID blockID) {
- GetKeyRequestProto.Builder readKeyRequest = GetKeyRequestProto.newBuilder()
+ private ContainerCommandRequestProto getGetBlockCommand(BlockID blockID) {
+ GetBlockRequestProto.Builder readBlockRequest =
+ GetBlockRequestProto.newBuilder()
.setBlockID(blockID.getDatanodeBlockIDProtobuf());
ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
.newBuilder()
- .setCmdType(ContainerProtos.Type.GetKey)
+ .setCmdType(ContainerProtos.Type.GetBlock)
.setContainerID(blockID.getContainerID())
.setTraceID(getBlockTraceID(blockID))
.setDatanodeUuid(datanodeUuid)
- .setGetKey(readKeyRequest);
+ .setGetBlock(readBlockRequest);
return request.build();
}
- private ContainerProtos.KeyData getKeyData(
+ private ContainerProtos.BlockData getBlockData(
BlockID blockID, String chunkKey) {
- ContainerProtos.KeyData.Builder builder = ContainerProtos.KeyData
+ ContainerProtos.BlockData.Builder builder = ContainerProtos.BlockData
.newBuilder()
.setBlockID(blockID.getDatanodeBlockIDProtobuf())
.addChunks(getChunkInfo(blockID, chunkKey));
@@ -275,16 +275,16 @@ public class BenchMarkDatanodeDispatcher {
}
@Benchmark
- public void putKey(BenchMarkDatanodeDispatcher bmdd) {
+ public void putBlock(BenchMarkDatanodeDispatcher bmdd) {
BlockID blockID = getRandomBlockID();
String chunkKey = getNewChunkToWrite();
- bmdd.dispatcher.dispatch(getPutKeyCommand(blockID, chunkKey));
+ bmdd.dispatcher.dispatch(getPutBlockCommand(blockID, chunkKey));
}
@Benchmark
- public void getKey(BenchMarkDatanodeDispatcher bmdd) {
+ public void getBlock(BenchMarkDatanodeDispatcher bmdd) {
BlockID blockID = getRandomBlockID();
- bmdd.dispatcher.dispatch(getGetKeyCommand(blockID));
+ bmdd.dispatcher.dispatch(getGetBlockCommand(blockID));
}
// Chunks writes from benchmark only reaches certain containers
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-7337. Expose per-node
over-allocation info in Node Report Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcb3709a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcb3709a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcb3709a
Branch: refs/heads/YARN-1011
Commit: dcb3709a2770f719f1fc5646e049767111d802d9
Parents: 33dcc79
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 08:39:03 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:28:47 2018 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/TestRMNMInfo.java | 14 ++-
.../hadoop/yarn/api/records/NodeReport.java | 98 +++++++++++++++-----
.../src/main/proto/yarn_protos.proto | 7 +-
.../applications/distributedshell/Client.java | 2 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 48 +++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 43 ++++++---
.../api/records/impl/pb/NodeReportPBImpl.java | 94 +++++++++++++++----
.../hadoop/yarn/server/utils/BuilderUtils.java | 37 ++++----
.../server/resourcemanager/ClientRMService.java | 23 +++--
.../resourcemanager/DefaultAMSProcessor.java | 22 +++--
.../yarn/server/resourcemanager/RMNMInfo.java | 45 +++++----
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../scheduler/SchedulerNodeReport.java | 50 +++++++---
.../webapp/dao/FifoSchedulerInfo.java | 9 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 17 ++--
.../server/resourcemanager/NodeManager.java | 4 +-
.../capacity/TestApplicationPriority.java | 66 ++++++++-----
.../capacity/TestCapacityScheduler.java | 32 ++++---
.../TestCapacitySchedulerMultiNodes.java | 10 +-
.../TestNodeLabelContainerAllocation.java | 40 ++++----
.../scheduler/fifo/TestFifoScheduler.java | 50 ++++++----
.../webapp/TestRMWebServicesNodes.java | 13 +--
22 files changed, 498 insertions(+), 228 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
index efea709..76b5493 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
@@ -124,11 +124,17 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected number of used containers",
- 0, n.get("NumContainers").asInt());
+ n.get("NodeId") + ": Unexpected number of guaranteed containers used",
+ 0, n.get("NumContainers").asInt());
+ Assert.assertEquals(n.get("NodeId") +
+ ": Unexpected number of opportunistic containers used",
+ 0, n.get("NumOpportunisticContainers").asInt());
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected amount of used memory",
+ n.get("NodeId") + ": Unexpected amount of guaranteed memory used",
0, n.get("UsedMemoryMB").asInt());
+ Assert.assertEquals(
+ n.get("NodeId") + ": Unexpected amount of used opportunistic memory",
+ 0, n.get("UsedOpportunisticMemoryMB").asInt());
Assert.assertNotNull(n.get("AvailableMemoryMB"));
}
}
@@ -161,6 +167,8 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
+ Assert.assertNull(n.get("NumOpportunisticContainers"));
+ Assert.assertNull(n.get("UsedOpportunisticMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 625ad23..dfd9607 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link NodeId} of the node.</li>
* <li>HTTP Tracking URL of the node.</li>
* <li>Rack name for the node.</li>
- * <li>Used {@link Resource} on the node.</li>
+ * <li>Used guaranteed {@link Resource} on the node.</li>
+ * <li>Used opportunistic {@link Resource} on the node.</li>
* <li>Total available {@link Resource} of the node.</li>
- * <li>Number of running containers on the node.</li>
+ * <li>Number of total running containers on the node.</li>
+ * <li>Number of running guaranteed containers on the node.</li>
+ * <li>Number of running opportunistic containers on the node.</li>
* </ul>
*
* @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
@@ -50,33 +53,37 @@ public abstract class NodeReport {
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newInstance(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime) {
+ return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed,
+ capability, numGuaranteedContainers, healthReport, lastHealthReportTime,
+ null, null, null, null, 0);
}
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType,
+ Resource opportunisticUsed, int numOpportunisticContainers) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setGuaranteedResourceUsed(guaranteedUsed);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
nodeReport.setDecommissioningTimeout(decommissioningTimeout);
nodeReport.setNodeUpdateType(nodeUpdateType);
+ nodeReport.setOpportunisticResourceUsed(opportunisticUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
return nodeReport;
}
@@ -129,17 +136,43 @@ public abstract class NodeReport {
public abstract void setRackName(String rackName);
/**
- * Get <em>used</em> <code>Resource</code> on the node.
- * @return <em>used</em> <code>Resource</code> on the node
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsed();
@Private
@Unstable
+ @Deprecated
public abstract void setUsed(Resource used);
-
+
+ /**
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourceUsed(Resource guaranteed);
+
+ /**
+ * Get <em>opportunistic</em> <code>Resource</code> used on the node.
+ * @return <em>opportunistic</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceUsed(Resource opportunistic);
+
/**
* Get the <em>total</em> <code>Resource</code> on the node.
* @return <em>total</em> <code>Resource</code> on the node
@@ -153,19 +186,40 @@ public abstract class NodeReport {
public abstract void setCapability(Resource capability);
/**
- * Get the <em>number of allocated containers</em> on the node.
- * @return <em>number of allocated containers</em> on the node
+ * Get the <em>number of guaranteed containers</em> allocated on the node.
+ * @return <em>number of guaranteed containers</em> allocated on the node
*/
@Private
@Unstable
- public abstract int getNumContainers();
+ public abstract int getNumGuaranteedContainers();
@Private
@Unstable
- public abstract void setNumContainers(int numContainers);
-
+ public abstract void setNumGuaranteedContainers(int numContainers);
- /**
+ /**
+ * Get the <em>number of opportunistic containers</em> allocated on the node.
+ * @return <em>number of opportunistic containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public abstract int getNumOpportunisticContainers();
+
+ @Private
+ @Unstable
+ public abstract void setNumOpportunisticContainers(int numContainers);
+
+ /**
+ * Get the <em>number of containers</em> allocated on the node.
+ * @return <em>number of containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public int getNumTotalContainers() {
+ return getNumGuaranteedContainers() + getNumOpportunisticContainers();
+ }
+
+ /**
* Get the <em>diagnostic health report</em> of the node.
* @return <em>diagnostic health report</em> of the node
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5fe2cc9..49bf895 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -344,9 +344,9 @@ message NodeReportProto {
optional NodeIdProto nodeId = 1;
optional string httpAddress = 2;
optional string rackName = 3;
- optional ResourceProto used = 4;
+ optional ResourceProto guaranteedResourceUsed = 4;
optional ResourceProto capability = 5;
- optional int32 numContainers = 6;
+ optional int32 numGuaranteedContainers = 6;
optional NodeStateProto node_state = 7;
optional string health_report = 8;
optional int64 last_health_report_time = 9;
@@ -356,6 +356,9 @@ message NodeReportProto {
optional uint32 decommissioning_timeout = 13;
optional NodeUpdateTypeProto node_update_type = 14;
repeated NodeAttributeProto node_attributes = 15;
+ optional int32 numOpportunisticContainers = 16;
+ optional ResourceProto opportunisticResourceUsed = 17;
+ optional int32 numTotalContainers = 18;
}
message NodeIdToLabelsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index e8b69fe..60a8a08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -647,7 +647,7 @@ public class Client {
+ ", nodeId=" + node.getNodeId()
+ ", nodeAddress=" + node.getHttpAddress()
+ ", nodeRackName=" + node.getRackName()
- + ", nodeNumContainers=" + node.getNumContainers());
+ + ", nodeNumContainers=" + node.getNumGuaranteedContainers());
}
QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index 44e9870..9429a18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
@Private
@@ -184,7 +185,7 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
- .getNumContainers());
+ .getNumGuaranteedContainers());
}
writer.flush();
}
@@ -209,13 +210,18 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
nodeReport.getNodeState(), nodeReport.getHttpAddress(),
- nodeReport.getNumContainers());
+ nodeReport.getNumGuaranteedContainers());
writer.println("Detailed Node Information :");
writer.print("\tConfigured Resources : ");
writer.println(nodeReport.getCapability());
- writer.print("\tAllocated Resources : ");
- if (nodeReport.getUsed() != null) {
- writer.print(nodeReport.getUsed());
+ writer.print("\tAllocated Guaranteed Resources : ");
+ if (nodeReport.getGuaranteedResourceUsed() != null) {
+ writer.print(nodeReport.getGuaranteedResourceUsed());
+ }
+ writer.println();
+ writer.print("\tAllocated Opportunistic Resources : ");
+ if (nodeReport.getOpportunisticResourceUsed() != null) {
+ writer.print(nodeReport.getOpportunisticResourceUsed());
}
writer.println();
@@ -286,16 +292,32 @@ public class NodeCLI extends YarnCLI {
nodeReportStr.print("\tHealth-Report : ");
nodeReportStr
.println(nodeReport.getHealthReport());
- nodeReportStr.print("\tContainers : ");
- nodeReportStr.println(nodeReport.getNumContainers());
- nodeReportStr.print("\tMemory-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
- : (nodeReport.getUsed().getMemorySize() + "MB"));
+ nodeReportStr.print("\tGuaranteed Containers : ");
+ nodeReportStr.println(nodeReport.getNumGuaranteedContainers());
+ nodeReportStr.print("\tOpportunistic Containers : ");
+ nodeReportStr.println(nodeReport.getNumOpportunisticContainers());
+ Resource guaranteedResourceUsed =
+ nodeReport.getGuaranteedResourceUsed();
+ Resource opportunisticResourceUsed =
+ nodeReport.getOpportunisticResourceUsed();
+ nodeReportStr.print("\tGuaranteed-Memory-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0MB" :
+ (guaranteedResourceUsed.getMemorySize() + "MB"));
+ nodeReportStr.print("\tOpportunistic-Memory-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0MB" :
+ (opportunisticResourceUsed.getMemorySize() + "MB"));
nodeReportStr.print("\tMemory-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
- nodeReportStr.print("\tCPU-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
- : (nodeReport.getUsed().getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tGuaranteed-CPU-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0 vcores" :
+ (guaranteedResourceUsed.getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tOpportunistic-CPU-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0 vcores" :
+ (opportunisticResourceUsed.getVirtualCores() + " vcores"));
nodeReportStr.print("\tCPU-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.print("\tNode-Labels : ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index a600895..6a24837 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1355,7 +1355,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1363,7 +1364,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1564,10 +1566,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tNode Attributes : rm.yarn.io/GPU(STRING)=ARM");
@@ -1602,10 +1607,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : ");
pw.println("\tNode Attributes : ");
@@ -1639,10 +1647,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tNode Attributes : ");
@@ -2078,10 +2089,12 @@ public class TestYarnCLI {
// ordered
nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
}
- NodeReport nodeReport = NodeReport.newInstance(NodeId
- .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
- "rack1", Records.newRecord(Resource.class), Records
- .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
+ "rack1", Records.newRecord(Resource.class),
+ Records.newRecord(Resource.class), 0, "",
+ 0, nodeLabels, null, null,
+ Records.newRecord(Resource.class), 0);
if (!emptyResourceUtilization) {
ResourceUtilization containersUtilization = ResourceUtilization
.newInstance(1024, 2048, 4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index 7d5a06a..ec0560e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -49,7 +49,8 @@ public class NodeReportPBImpl extends NodeReport {
private NodeReportProto.Builder builder = null;
private boolean viaProto = false;
private NodeId nodeId;
- private Resource used;
+ private Resource guaranteedResourceUsed;
+ private Resource opportunisticResourceUsed;
private Resource capability;
private ResourceUtilization containersUtilization = null;
private ResourceUtilization nodeUtilization = null;
@@ -114,9 +115,17 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public int getNumContainers() {
+ public int getNumGuaranteedContainers() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- return (p.hasNumContainers()) ? p.getNumContainers() : 0;
+ return (p.hasNumGuaranteedContainers()) ?
+ p.getNumGuaranteedContainers() : 0;
+ }
+
+ @Override
+ public int getNumOpportunisticContainers() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasNumOpportunisticContainers()) ?
+ p.getNumOpportunisticContainers() : 0;
}
@Override
@@ -125,18 +134,25 @@ public class NodeReportPBImpl extends NodeReport {
return (p.hasRackName()) ? p.getRackName() : null;
}
+ @Deprecated
@Override
public Resource getUsed() {
- if (this.used != null) {
- return this.used;
+ return getGuaranteedResourceUsed();
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsed() {
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
}
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUsed()) {
+ if (!p.hasOpportunisticResourceUsed()) {
return null;
}
- this.used = convertFromProtoFormat(p.getUsed());
- return this.used;
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getOpportunisticResourceUsed());
+ return this.opportunisticResourceUsed;
}
@Override
@@ -201,13 +217,24 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public void setNumContainers(int numContainers) {
+ public void setNumGuaranteedContainers(int numContainers) {
+ maybeInitBuilder();
+ if (numContainers == 0) {
+ builder.clearNumGuaranteedContainers();
+ return;
+ }
+ builder.setNumGuaranteedContainers(numContainers);
+ }
+
+ @Override
+ public void setNumOpportunisticContainers(int numContainers) {
maybeInitBuilder();
if (numContainers == 0) {
- builder.clearNumContainers();
+ builder.clearNumOpportunisticContainers();
return;
}
- builder.setNumContainers(numContainers);
+ builder.setNumOpportunisticContainers(numContainers);
+
}
@Override
@@ -220,13 +247,43 @@ public class NodeReportPBImpl extends NodeReport {
builder.setRackName(rackName);
}
+ @Deprecated
@Override
public void setUsed(Resource used) {
+ setGuaranteedResourceUsed(used);
+ }
+
+ @Override
+ public Resource getGuaranteedResourceUsed() {
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasGuaranteedResourceUsed()) {
+ return null;
+ }
+ this.guaranteedResourceUsed =
+ convertFromProtoFormat(p.getGuaranteedResourceUsed());
+ return this.guaranteedResourceUsed;
+ }
+
+ @Override
+ public void setGuaranteedResourceUsed(Resource guaranteed) {
+ maybeInitBuilder();
+ if (guaranteedResourceUsed == null) {
+ builder.clearGuaranteedResourceUsed();
+ }
+ this.guaranteedResourceUsed = guaranteed;
+ }
+
+ @Override
+ public void setOpportunisticResourceUsed(Resource opportunisticUsed) {
maybeInitBuilder();
- if (used == null) {
- builder.clearUsed();
+ if (opportunisticUsed == null) {
+ builder.clearOpportunisticResourceUsed();
}
- this.used = used;
+ this.opportunisticResourceUsed = opportunisticUsed;
}
public NodeReportProto getProto() {
@@ -263,8 +320,13 @@ public class NodeReportPBImpl extends NodeReport {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.used != null) {
- builder.setUsed(convertToProtoFormat(this.used));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setGuaranteedResourceUsed(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setOpportunisticResourceUsed(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.capability != null) {
builder.setCapability(convertToProtoFormat(this.capability));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index b9f35a5..ff88225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -186,26 +186,23 @@ public class BuilderUtils {
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainer,
+ String healthReport, long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) {
+ return newNodeReport(nodeId, nodeState, httpAddress, rackName,
+ guaranteedResourceUsed, capability, numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainer, healthReport,
+ lastHealthReportTime, nodeLabels, null, null,
+ decommissioningTimeout, nodeUpdateType, null);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- nodeLabels, null, null, decommissioningTimeout, nodeUpdateType, null);
- }
-
- public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainers,
+ String healthReport, long lastHealthReportTime,
Set<String> nodeLabels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
NodeUpdateType nodeUpdateType, Set<NodeAttribute> attrs) {
@@ -214,9 +211,11 @@ public class BuilderUtils {
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
+ nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
+ nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 4075332..1d3fef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1045,18 +1045,27 @@ public class ClientRMService extends AbstractService implements
private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
- }
+ guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
+ }
Set<NodeAttribute> attrs = rmNode.getAllNodeAttributes();
NodeReport report =
BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed,
+ rmNode.getTotalCapability(), numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index b2c5ef3..ddfabdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -365,17 +365,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
RMNode rmNode = rmNodeEntry.getKey();
SchedulerNodeReport schedulerNodeReport =
getScheduler().getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ guaranteedResourceUsed =
+ schedulerNodeReport.getGuaranteedResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeId nodeId = rmNode.getNodeID();
NodeReport report =
BuilderUtils.newNodeReport(nodeId, rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed, rmNode.getTotalCapability(),
+ numGuaranteedContainers, opportunisticResourceUsed,
+ numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
rmNodeEntry.getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..849db5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans {
List<InfoMap> nodesInfo = new ArrayList<InfoMap>();
for (final RMNode ni : nodes) {
- SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
- InfoMap info = new InfoMap();
- info.put("HostName", ni.getHostName());
- info.put("Rack", ni.getRackName());
- info.put("State", ni.getState().toString());
- info.put("NodeId", ni.getNodeID());
- info.put("NodeHTTPAddress", ni.getHttpAddress());
- info.put("LastHealthUpdate",
- ni.getLastHealthReportTime());
- info.put("HealthReport",
- ni.getHealthReport());
- info.put("NodeManagerVersion",
- ni.getNodeManagerVersion());
- if(report != null) {
- info.put("NumContainers", report.getNumContainers());
- info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
- info.put("AvailableMemoryMB",
- report.getAvailableResource().getMemorySize());
- }
+ SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
+ InfoMap info = new InfoMap();
+ info.put("HostName", ni.getHostName());
+ info.put("Rack", ni.getRackName());
+ info.put("State", ni.getState().toString());
+ info.put("NodeId", ni.getNodeID());
+ info.put("NodeHTTPAddress", ni.getHttpAddress());
+ info.put("LastHealthUpdate",
+ ni.getLastHealthReportTime());
+ info.put("HealthReport",
+ ni.getHealthReport());
+ info.put("NodeManagerVersion",
+ ni.getNodeManagerVersion());
+ if(report != null) {
+ info.put("NumContainers", report.getNumGuaranteedContainers());
+ info.put("NumOpportunisticContainers",
+ report.getNumOpportunisticContainers());
+ info.put("UsedMemoryMB",
+ report.getGuaranteedResourceUsed().getMemorySize());
+ info.put("UsedOpportunisticMemoryMB",
+ report.getOpportunisticResourceUsed().getMemorySize());
+ info.put("AvailableMemoryMB",
+ report.getAvailableGuaranteedResource().getMemorySize());
+ }
- nodesInfo.add(info);
+ nodesInfo.add(info);
}
return JSON.toString(nodesInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b53b32b..0e32f1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -142,7 +142,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/** Physical resources in the node. */
private volatile Resource physicalResource;
- /* Container Queue Information for the node.. Used by Distributed Scheduler */
+ /* Container Queue Information for the node..*/
private OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerAllocationExpirer containerAllocationExpirer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index ea30d78..25621a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Stable
public class SchedulerNodeReport {
- private final Resource used;
- private final Resource avail;
- private final int num;
+ private final Resource guaranteedResourceUsage;
+ private final Resource opportunisticResourceUsage;
+ private final Resource guaranteedResourceAvail;
+ private final int numOfGuaranteedContainers;
+ private final int numOfOpportunisticContainers;
public SchedulerNodeReport(SchedulerNode node) {
- this.used = node.getAllocatedResource();
- this.avail = node.getUnallocatedResource();
- this.num = node.getNumGuaranteedContainers();
+ this.guaranteedResourceUsage = node.getAllocatedResource();
+ this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated();
+ this.guaranteedResourceAvail = node.getUnallocatedResource();
+ this.numOfGuaranteedContainers = node.getNumGuaranteedContainers();
+ this.numOfOpportunisticContainers = node.getNumOpportunisticContainers();
}
/**
- * @return the amount of resources currently used by the node.
+ * @return the amount of guaranteed resources currently used by the node.
*/
- public Resource getUsedResource() {
- return used;
+ public Resource getGuaranteedResourceUsed() {
+ return guaranteedResourceUsage;
}
/**
- * @return the amount of resources currently available on the node
+ * @return the amount of opportunistic resources currently used by the node.
*/
- public Resource getAvailableResource() {
- return avail;
+ public Resource getOpportunisticResourceUsed() {
+ return opportunisticResourceUsage;
}
/**
- * @return the number of containers currently running on this node.
+ * @return the amount of guaranteed resources currently available on the node
*/
- public int getNumContainers() {
- return num;
+ public Resource getAvailableGuaranteedResource() {
+ return guaranteedResourceAvail;
+ }
+
+ /**
+ * @return the number of guaranteed containers currently running on
+ * this node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numOfGuaranteedContainers;
+ }
+
+ /**
+ * @return the number of opportunistic containers currently running on
+ * this node.
+ */
+ public int getNumOpportunisticContainers() {
+ return numOfOpportunisticContainers;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
index 1752546..f5d8d07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
@@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo {
for (RMNode ni : rmContext.getRMNodes().values()) {
SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
- this.usedNodeCapacity += report.getUsedResource().getMemorySize();
- this.availNodeCapacity += report.getAvailableResource().getMemorySize();
+ this.usedNodeCapacity +=
+ report.getGuaranteedResourceUsed().getMemorySize();
+ this.availNodeCapacity +=
+ report.getAvailableGuaranteedResource().getMemorySize();
this.totalNodeCapacity += ni.getTotalCapability().getMemorySize();
- this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+ this.numContainers +=
+ fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 7e5d20a..5618a98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -75,14 +75,17 @@ public class NodeInfo {
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null) {
- this.numContainers = report.getNumContainers();
- this.usedMemoryMB = report.getUsedResource().getMemorySize();
- this.availMemoryMB = report.getAvailableResource().getMemorySize();
- this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.numContainers = report.getNumGuaranteedContainers();
+ this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize();
+ this.availMemoryMB =
+ report.getAvailableGuaranteedResource().getMemorySize();
+ this.usedVirtualCores =
+ report.getGuaranteedResourceUsed().getVirtualCores();
this.availableVirtualCores =
- report.getAvailableResource().getVirtualCores();
- this.usedResource = new ResourceInfo(report.getUsedResource());
- this.availableResource = new ResourceInfo(report.getAvailableResource());
+ report.getAvailableGuaranteedResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed());
+ this.availableResource =
+ new ResourceInfo(report.getAvailableGuaranteedResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index ee974e3..96613fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol {
LOG.info("Checking resource usage for " + containerManagerAddress);
Assert.assertEquals(available.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getAvailableResource().getMemorySize());
+ this.nodeId).getAvailableGuaranteedResource().getMemorySize());
Assert.assertEquals(used.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getUsedResource().getMemorySize());
+ this.nodeId).getGuaranteedResourceUsed().getMemorySize());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index cad0151..9c317e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -176,8 +176,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -189,8 +191,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -210,8 +214,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// send updated request for App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -226,8 +232,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -261,8 +269,10 @@ public class TestApplicationPriority {
// check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(8 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 7
Priority appPriority2 = Priority.newInstance(7);
@@ -288,8 +298,10 @@ public class TestApplicationPriority {
// check node report, 1 GB used and 7 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(7 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -486,8 +498,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -499,8 +513,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -522,8 +538,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -535,8 +553,10 @@ public class TestApplicationPriority {
Assert.assertEquals(2, allocated2.size());
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// kill 1 more
counter = 0;
@@ -552,8 +572,10 @@ public class TestApplicationPriority {
// check node report, 14 GB used and 2 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(14 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Change the priority of App1 to 3 (lowest)
Priority appPriority3 = Priority.newInstance(3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 9e15856..def0419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -1246,8 +1246,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1268,8 +1270,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1288,7 +1292,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
waitCount = 0;
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- if (report_nm1.getAvailableResource().getMemorySize() != 0) {
+ if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1297,8 +1301,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
}
// Now, the used resource is still 4 GB, and available resource is minus value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
@@ -1314,9 +1320,11 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Verify no NPE is trigger in schedule after resource is updated.
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
@@ -2816,8 +2824,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(9 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
index c90af94..f5b73ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
@@ -123,9 +123,10 @@ public class TestCapacitySchedulerMultiNodes extends CapacitySchedulerTestBase {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(2 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(8 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Ideally thread will invoke this, but thread operates every 1sec.
// Hence forcefully recompute nodes.
@@ -137,9 +138,10 @@ public class TestCapacitySchedulerMultiNodes extends CapacitySchedulerTestBase {
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
// Ideally thread will invoke this, but thread operates every 1sec.
// Hence forcefully recompute nodes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 02c0cc5..8023457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(5 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(5 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB());
@@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(3 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(7 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
double delta = 0.0001;
@@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(14 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Try to launch app2 in a2, asked 2GB, should success
// app2 -> a2
@@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation {
reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a2
cs.killAllAppsInQueue("a2");
@@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
cs.getApplicationAttempt(am3.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a1
cs.killAllAppsInQueue("a1");
@@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
cs.getApplicationAttempt(am4.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
rm1.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 3f8a552..ad68e31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -701,7 +701,8 @@ public class TestFifoScheduler {
am1.registerAppAttempt();
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
RMApp app2 = rm.submitApp(2048);
// kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -711,7 +712,8 @@ public class TestFifoScheduler {
am2.registerAppAttempt();
SchedulerNodeReport report_nm2 =
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
@@ -747,11 +749,15 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getAvailableGuaranteedResource().getMemorySize());
- Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemorySize());
@@ -769,7 +775,8 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ 5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -826,7 +833,8 @@ public class TestFifoScheduler {
int checkAlloc =
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -1117,8 +1125,10 @@ public class TestFifoScheduler {
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1139,8 +1149,10 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1159,7 +1171,7 @@ public class TestFifoScheduler {
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
if (null != report_nm1 &&
- report_nm1.getAvailableResource().getMemorySize() != 0) {
+ report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1169,8 +1181,10 @@ public class TestFifoScheduler {
// Now, the used resource is still 4 GB, and available resource is minus
// value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource
// over-commitment.
@@ -1188,9 +1202,11 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcb3709a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 72b7a45..057c5ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -817,15 +817,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
if (report != null) {
assertEquals("numContainers doesn't match: " + numContainers,
- report.getNumContainers(), numContainers);
+ report.getNumGuaranteedContainers(), numContainers);
assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report
- .getUsedResource().getMemorySize(), usedMemoryMB);
+ .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
- .getAvailableResource().getMemorySize(), availMemoryMB);
+ .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
- .getUsedResource().getVirtualCores(), usedVirtualCores);
- assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
- .getAvailableResource().getVirtualCores(), availVirtualCores);
+ .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores,
+ report.getAvailableGuaranteedResource().getVirtualCores(),
+ availVirtualCores);
}
if (opportunisticStatus != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-6750. Add a configuration to
cap how much a NM can be overallocated. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6750. Add a configuration to cap how much a NM can be overallocated. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/353bdbcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/353bdbcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/353bdbcb
Branch: refs/heads/YARN-1011
Commit: 353bdbcb3eea68067a470bf7dc5285edd7a34acd
Parents: d6fa51c
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 09:17:56 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:28:47 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 10 ++
.../scheduler/AbstractYarnScheduler.java | 4 +
.../scheduler/SchedulerNode.java | 42 ++++++--
.../scheduler/fair/FSSchedulerNode.java | 5 +
.../scheduler/fair/FairScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 107 +++++++++++++++++++
7 files changed, 168 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 71651a6..0326880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -398,6 +398,11 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
+ /** The global max overallocation per node in terms of their capacity. */
+ public static final String PER_NODE_MAX_OVERALLOCATION_RATIO =
+ RM_PREFIX + "overallocation.per-node-max-ratio";
+ public static final float DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO = 4.0f;
+
/** Setting that controls whether opportunistic container allocation
* is enabled or not. */
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b45fe06..bc9279e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3342,6 +3342,16 @@
<property>
<description>
+ The maximum amount of resources, specified as a ratio to node capacity,
+ that can be allocated to opportunistic containers on any given node in
+ the cluster.
+ </description>
+ <name>yarn.resourcemanager.overallocation.per-node-max-ratio</name>
+ <value>4.0</value>
+ </property>
+
+ <property>
+ <description>
Frequency for computing least loaded NMs.
</description>
<name>yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 6b1fdcb..278e474 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -119,6 +119,7 @@ public abstract class AbstractYarnScheduler
new ClusterNodeTracker<>();
protected Resource minimumAllocation;
+ protected float maxOverAllocationRatioPerNode;
protected volatile RMContext rmContext;
@@ -201,6 +202,9 @@ public abstract class AbstractYarnScheduler
nodeTracker.setConfiguredMaxAllocationWaitTime(
configuredMaximumAllocationWaitTime);
maxClusterLevelAppPriority = getMaxPriorityFromConf(conf);
+ maxOverAllocationRatioPerNode = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
createReleaseCache();
autoUpdateContainers =
conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index ff3640c..3e16dc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -94,6 +94,10 @@ public abstract class SchedulerNode {
protected Resource resourceAllocatedPendingLaunch =
Resource.newInstance(0, 0);
+ // The max amount of resources that can be allocated to opportunistic
+ // containers on the node, specified as a ratio to its capacity
+ private final float maxOverAllocationRatio;
+
private volatile Set<String> labels = null;
private volatile Set<NodeAttribute> nodeAttributes = null;
@@ -102,7 +106,7 @@ public abstract class SchedulerNode {
private volatile long lastHeartbeatMonotonicTime;
public SchedulerNode(RMNode node, boolean usePortForNodeName,
- Set<String> labels) {
+ Set<String> labels, float maxOverAllocationRatio) {
this.rmNode = node;
this.rmContext = node.getRMContext();
this.unallocatedResource = Resources.clone(node.getTotalCapability());
@@ -114,10 +118,24 @@ public abstract class SchedulerNode {
}
this.labels = ImmutableSet.copyOf(labels);
this.lastHeartbeatMonotonicTime = Time.monotonicNow();
+ this.maxOverAllocationRatio = maxOverAllocationRatio;
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> labels) {
+ this(node, usePortForNodeName, labels,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverAllocationRatio) {
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ maxOverAllocationRatio);
}
public SchedulerNode(RMNode node, boolean usePortForNodeName) {
- this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
}
public RMNode getRMNode() {
@@ -671,9 +689,11 @@ public abstract class SchedulerNode {
/**
* Get the amount of resources that can be allocated to opportunistic
- * containers in the case of overallocation. It is calculated as
+ * containers in the case of overallocation, calculated as
* node capacity - (node utilization + resources of allocated-yet-not-started
- * containers).
+ * containers), subject to the maximum amount of resources that can be
+ * allocated to opportunistic containers on the node specified as a ratio to
+ * its capacity.
* @return the amount of resources that are available to be allocated to
* opportunistic containers
*/
@@ -706,11 +726,21 @@ public abstract class SchedulerNode {
Resource resourceAllowedForOpportunisticContainers =
Resources.createResource(allowedMemory, allowedCpu);
- // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
- // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ // cap the total amount of resources allocated to OPPORTUNISTIC containers
+ Resource maxOverallocation = getMaxOverallocationAllowed();
+ Resources.subtractFrom(maxOverallocation, allocatedResourceOpportunistic);
+ resourceAllowedForOpportunisticContainers = Resources.componentwiseMin(
+ maxOverallocation, resourceAllowedForOpportunisticContainers);
+
return resourceAllowedForOpportunisticContainers;
}
+ private Resource getMaxOverallocationAllowed() {
+ long maxMemory = (long) (capacity.getMemorySize() * maxOverAllocationRatio);
+ int maxVcore = (int) (capacity.getVirtualCores() * maxOverAllocationRatio);
+ return Resource.newInstance(maxMemory, maxVcore);
+ }
+
private static class ContainerInfo {
private final RMContainer container;
private boolean launchedOnNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 95490f5..a53dda4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -66,10 +66,15 @@ public class FSSchedulerNode extends SchedulerNode {
// slated for preemption
private Resource totalResourcesPreempted = Resource.newInstance(0, 0);
+ @VisibleForTesting
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
}
+ public FSSchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverallocationRatio) {
+ super(node, usePortForNodeName, maxOverallocationRatio);
+ }
/**
* Total amount of reserved resources including reservations and preempted
* containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 5d48fac..25782a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -774,7 +774,7 @@ public class FairScheduler extends
writeLock.lock();
try {
FSSchedulerNode schedulerNode = new FSSchedulerNode(node,
- usePortForNodeName);
+ usePortForNodeName, maxOverAllocationRatioPerNode);
nodeTracker.addNode(schedulerNode);
triggerUpdate();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353bdbcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index fbb7243..9302f88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3143,6 +3143,113 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test that max overallocation per node is enforced by Fair Scheduler.
+ * @throws Exception
+ */
+ @Test
+ public void testMaxOverallocationPerNode() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ float maxOverallocationRatio = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO, 1.5f);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 1G of memory and 1 vcores and an overallocation
+ // threshold of 1.0f and 1.0f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(1f, 1f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024, 1), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is zero after the container runs
+ ContainerStatus containerStatus1 = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus1),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create a scheduling request that should get allocated an OPPORTUNISTIC
+ // container because the node utilization is zero
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // node utilization is still zero after the container runs
+ ContainerStatus containerStatus2 = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus2),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create another scheduling request that should not get any allocation
+ // because of the max overallocation on the node will be exceeded.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ maxOverallocationRatio);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-6670 Add separate NM
overallocation thresholds for cpu and memory (Haibo Chen)
Posted by ha...@apache.org.
YARN-6670 Add separate NM overallocation thresholds for cpu and memory (Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/013c977a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/013c977a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/013c977a
Branch: refs/heads/YARN-1011
Commit: 013c977a4b6f13dc17f6b7cff05bc963d467b3e0
Parents: 6389e0a
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jul 10 09:55:42 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 14:10:13 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 36 ++++++++--
.../src/main/resources/yarn-default.xml | 42 ++++++++++--
.../server/api/records/ResourceThresholds.java | 11 ++-
.../monitor/ContainersMonitorImpl.java | 71 +++++++++++++++-----
4 files changed, 126 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/013c977a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2e128a2..daf9910 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2115,17 +2115,39 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
3000;
- /** Overallocation (= allocation based on utilization) configs. */
- public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD =
- NM_PREFIX + "overallocation.allocation-threshold";
- public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD
- = 0f;
+ /**
+ * General overallocation threshold if no resource-type-specific
+ * threshold is provided.
+ */
+ public static final String NM_OVERALLOCATION_GENERAL_THRESHOLD =
+ NM_PREFIX + "overallocation.general-utilization-threshold";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_GENERAL_THRESHOLD = -1.0f;
+ /**
+ * The maximum value of utilization threshold for all resource types
+ * up to which the scheduler allocates OPPORTUNISTIC containers.
+ */
@Private
- public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f;
+ public static final float MAX_NM_OVERALLOCATION_THRESHOLD = 0.95f;
+
+ /**
+ * NM CPU utilization threshold up to which the scheduler allocates
+ * OPPORTUNISTIC containers after the node's capacity is fully allocated.
+ */
+ public static final String NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD =
+ NM_PREFIX + "overallocation.cpu-utilization-threshold";
+
+ /**
+ * NM memory utilization threshold up to which the scheduler allocates
+ * OPPORTUNISTIC containers after the node's capacity is fully allocated.
+ */
+ public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
+ NM_PREFIX + "overallocation.memory-utilization-threshold";
+
public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
NM_PREFIX + "overallocation.preemption-threshold";
public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
- = 0f;
+ = 0.96f;
/**
* Interval of time the linux container executor should try cleaning up
http://git-wip-us.apache.org/repos/asf/hadoop/blob/013c977a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8001580..84a4225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1727,14 +1727,44 @@
<property>
<description>The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node that
+ applies to all resource types (expressed as a float between 0 and 0.95).
+ By default, over-allocation is turned off (value = -1). When turned on,
+ the node allows running OPPORTUNISTIC containers when the aggregate
+ utilization for each resource type is under the value specified here
+ multiplied by the node's advertised capacity. Note for each resource type,
+ it can be overridden by the type specific setting:
+ yarn.nodemanager.overallocation.cpu-utilization-threshold for CPU
+ yarn.nodemanager.overallocation.memory-utilization-threshold for memory
+ </description>
+ <name>yarn.nodemanager.overallocation.general-utilization-threshold</name>
+ <value>-1.0</value>
+ </property>
+
+ <property>
+ <description>The extent of over-allocation (container-allocation based on
current utilization instead of prior allocation) allowed on this node,
- expressed as a float between 0 and 0.95. By default, over-allocation is
- turned off (value = 0). When turned on, the node allows running
- OPPORTUNISTIC containers when the aggregate utilization is under the
+ in terms of the percentage of overall NM memory capacity utilized (
+ expressed as a float between 0 and 0.95). By default, over-allocation is
+ turned off (value = -1). When turned on, the node allows running
+ OPPORTUNISTIC containers only when the aggregate utilization is under the
value specified here multiplied by the node's advertised capacity.
</description>
- <name>yarn.nodemanager.overallocation.allocation-threshold</name>
- <value>0f</value>
+ <name>yarn.nodemanager.overallocation.memory-utilization-threshold</name>
+ <value>${yarn.nodemanager.overallocation.general-utilization-threshold}</value>
+ </property>
+
+ <property>
+ <description>The extent of over-allocation (container-allocation based on
+ current utilization instead of prior allocation) allowed on this node,
+ in terms of the percentage of overall NM CPU capacity utilized (
+ expressed as a float between 0 and 0.95). By default, over-allocation is
+ turned off (value = -1). When turned on, the node allows running
+ OPPORTUNISTIC containers only when the aggregate utilization is under the
+ value specified here multiplied by the node's advertised capacity.
+ </description>
+ <name>yarn.nodemanager.overallocation.cpu-utilization-threshold</name>
+ <value>${yarn.nodemanager.overallocation.general-utilization-threshold}</value>
</property>
<property>
@@ -1743,7 +1773,7 @@
beyond which OPPORTUNISTIC containers should start getting preempted.
</description>
<name>yarn.nodemanager.overallocation.preemption-threshold</name>
- <value>1</value>
+ <value>0.96</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/013c977a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
index d57706a..c81e405 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java
@@ -28,10 +28,15 @@ import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImp
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class ResourceThresholds {
- public static ResourceThresholds newInstance(float threshold) {
+ public static ResourceThresholds newInstance(float overallThreshold) {
+ return newInstance(overallThreshold, overallThreshold);
+ }
+
+ public static ResourceThresholds newInstance(float cpuThreshold,
+ float memoryThreshold) {
ResourceThresholds thresholds = new ResourceThresholdsPBImpl();
- thresholds.setMemoryThreshold(threshold);
- thresholds.setCpuThreshold(threshold);
+ thresholds.setCpuThreshold(cpuThreshold);
+ thresholds.setMemoryThreshold(memoryThreshold);
return thresholds;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/013c977a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index cdd3476..d7c81bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -277,25 +277,60 @@ public class ContainersMonitorImpl extends AbstractService implements
}
private void initializeOverAllocation(Configuration conf) {
- float overAllocationTreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
- overAllocationTreshold = Math.min(overAllocationTreshold,
- YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD);
- overAllocationTreshold = Math.max(0, overAllocationTreshold);
-
- if (overAllocationTreshold > 0f) {
- ((NodeManager.NMContext) context).setOverAllocationInfo(
- OverAllocationInfo.newInstance(
- ResourceThresholds.newInstance(overAllocationTreshold)));
-
- float preemptionThreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
-
- this.overAllocationPreemptionThresholds =
- ResourceThresholds.newInstance(preemptionThreshold);
+ float generalResourceOverAllocationThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_GENERAL_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_GENERAL_THRESHOLD);
+
+ float overAllocationMemoryUtilizationThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
+ generalResourceOverAllocationThreshold);
+ overAllocationMemoryUtilizationThreshold = Math.min(
+ overAllocationMemoryUtilizationThreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
+ if (overAllocationMemoryUtilizationThreshold <= 0) {
+ LOG.info("NodeManager oversubscription is disabled because the memory " +
+ "utilization threshold is no larger than zero.");
+ return;
+ }
+
+ float overAllocationCpuUtilizationThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
+ generalResourceOverAllocationThreshold);
+ overAllocationCpuUtilizationThreshold = Math.min(
+ overAllocationCpuUtilizationThreshold,
+ YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
+ if (overAllocationCpuUtilizationThreshold <= 0) {
+ LOG.info("NodeManager oversubscription is disabled because the CPU " +
+ "utilization threshold is no larger than zero.");
+ return;
+ }
+
+ float preemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
+ if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because preemption" +
+ "threshold is no larger than the cpu utilization threshold.");
+ return;
}
+ if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because preemption" +
+ "threshold is no larger than the memory utilization threshold.");
+ return;
+ }
+
+ ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
+ overAllocationCpuUtilizationThreshold,
+ overAllocationMemoryUtilizationThreshold);
+ ((NodeManager.NMContext) context).setOverAllocationInfo(
+ OverAllocationInfo.newInstance(resourceThresholds));
+ this.overAllocationPreemptionThresholds =
+ ResourceThresholds.newInstance(preemptionThreshold);
+
+ LOG.info("NodeManager oversubscription enabled with overallocation " +
+ "thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
+ ", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
+ " threshold: " + preemptionThreshold);
}
private boolean isResourceCalculatorAvailable() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-8809. Refactor
AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container
completion codepaths. (Haibo Chen via asuresh)
Posted by ha...@apache.org.
YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo Chen via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/954986dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/954986dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/954986dd
Branch: refs/heads/YARN-1011
Commit: 954986dd5201368b553bfc99d8a099e9696d3e3c
Parents: 124b378
Author: Arun Suresh <as...@apache.org>
Authored: Fri Sep 21 12:02:00 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:48:55 2018 -0700
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 32 ++++++++------------
.../scheduler/capacity/CapacityScheduler.java | 25 ++++++++++++++-
.../scheduler/fair/FairScheduler.java | 11 ++++---
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 25 +++++++++++++++
5 files changed, 70 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/954986dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a8f5093..63f534f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -677,25 +677,12 @@ public abstract class AbstractYarnScheduler
}
if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
- completedContainerInternal(rmContainer, containerStatus, event);
+ completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
completeOustandingUpdatesWhichAreReserved(
rmContainer, containerStatus, event);
} else {
- ContainerId containerId = rmContainer.getContainerId();
- // Inform the container
- rmContainer.handle(
- new RMContainerFinishedEvent(containerId, containerStatus, event));
- SchedulerApplicationAttempt schedulerAttempt =
- getCurrentAttemptForContainer(containerId);
- if (schedulerAttempt != null) {
- schedulerAttempt.removeRMContainer(containerId);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Completed container: " + rmContainer.getContainerId() +
- " in state: " + rmContainer.getState() + " event:" + event);
- }
- getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
- rmContainer.getContainerId(), false);
+ completeOpportunisticContainerInternal(rmContainer, containerStatus,
+ event);
}
// If the container is getting killed in ACQUIRED state, the requester (AM
@@ -705,6 +692,12 @@ public abstract class AbstractYarnScheduler
recoverResourceRequestForContainer(rmContainer);
}
+ protected void completeOpportunisticContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
+ }
+
// Optimization:
// Check if there are in-flight container updates and complete the
// associated temp containers. These are removed when the app completes,
@@ -722,7 +715,7 @@ public abstract class AbstractYarnScheduler
.getReservedSchedulerKey().getContainerToUpdate();
if (containerToUpdate != null &&
containerToUpdate.equals(containerStatus.getContainerId())) {
- completedContainerInternal(resContainer,
+ completeGuaranteedContainerInternal(resContainer,
ContainerStatus.newInstance(resContainer.getContainerId(),
containerStatus.getState(), containerStatus
.getDiagnostics(),
@@ -732,8 +725,9 @@ public abstract class AbstractYarnScheduler
}
}
- // clean up a completed container
- protected abstract void completedContainerInternal(RMContainer rmContainer,
+ // clean up a completed guaranteed container
+ protected abstract void completeGuaranteedContainerInternal(
+ RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event);
protected void releaseContainers(List<ContainerId> containers,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/954986dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 0b7115b..c06122f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -2066,7 +2067,29 @@ public class CapacityScheduler extends
}
@Override
- protected void completedContainerInternal(
+ protected void completeOpportunisticContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ ContainerId containerId = rmContainer.getContainerId();
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerFinishedEvent(containerId, containerStatus, event));
+ SchedulerApplicationAttempt schedulerAttempt =
+ getCurrentAttemptForContainer(containerId);
+ if (schedulerAttempt != null) {
+ schedulerAttempt.removeRMContainer(containerId);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Completed OPPORTUNISTIC container: " +
+ rmContainer.getContainerId() + " in state: " +
+ rmContainer.getState() + " event:" + event);
+ }
+ getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
+ rmContainer.getContainerId(), false);
+ }
+
+ @Override
+ protected void completeGuaranteedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
Container container = rmContainer.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/954986dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 31f1961..744776a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -719,11 +719,14 @@ public class FairScheduler extends
}
}
- /**
- * Clean up a completed container.
- */
@Override
- protected void completedContainerInternal(
+ protected void completeGuaranteedContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ completeContainerInternal(rmContainer, containerStatus, event);
+ }
+
+ private void completeContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
writeLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/954986dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 8396db5..76648c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -836,7 +836,7 @@ public class FifoScheduler extends
@Lock(FifoScheduler.class)
@Override
- protected synchronized void completedContainerInternal(
+ protected synchronized void completeGuaranteedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/954986dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 7fbf84a..1ac20f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3194,6 +3194,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("unexpected container execution type",
ExecutionType.GUARANTEED,
allocatedContainers1.get(0).getExecutionType());
+ assertEquals(1,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
// node utilization is low after the container is launched on the node
ContainerStatus containerStatus = ContainerStatus.newInstance(
@@ -3235,6 +3237,29 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// OPPORTUNISTIC container allocation.
assertTrue("No reservation should be made.",
scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ assertEquals(3,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
+ assertEquals(3200 + 512 + 1024,
+ scheduler.getRootQueueMetrics().getAllocatedMB());
+ assertEquals(3,
+ scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
+
+ // now the OPPORTUNISTIC container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers3.get(0).getId(),
+ ContainerState.COMPLETE, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ assertEquals(2,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
+ assertEquals(3200 + 512,
+ scheduler.getRootQueueMetrics().getAllocatedMB());
+ assertEquals(2,
+ scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
} finally {
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: MAPREDUCE-6926. Allow MR jobs to
opt out of oversubscription. Contributed by Haibo Chen.
Posted by ha...@apache.org.
MAPREDUCE-6926. Allow MR jobs to opt out of oversubscription. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f00f7ad9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f00f7ad9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f00f7ad9
Branch: refs/heads/YARN-1011
Commit: f00f7ad904cc6abad3294fbf1bca3b9f0908e218
Parents: 979420f
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jan 10 13:21:11 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:39:11 2018 -0700
----------------------------------------------------------------------
.../v2/app/rm/RMContainerRequestor.java | 48 ++---
.../v2/app/rm/TestRMContainerAllocator.java | 192 +++++++++++++++++++
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +
.../src/main/resources/mapred-default.xml | 8 +
4 files changed, 231 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f00f7ad9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index bb3e1fa..d996690 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -111,6 +111,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
private final Set<String> blacklistRemovals = Collections
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private boolean optOutOfOversubscription;
public RMContainerRequestor(ClientService clientService, AppContext context) {
super(clientService, context);
@@ -136,9 +137,11 @@ public abstract class RMContainerRequestor extends RMCommunicator {
public ContainerRequest(ContainerRequestEvent event, Priority priority,
String nodeLabelExpression) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
- event.getRacks(), priority, nodeLabelExpression);
+ event.getRacks(), priority, System.currentTimeMillis(),
+ nodeLabelExpression);
}
+ @VisibleForTesting
public ContainerRequest(ContainerRequestEvent event, Priority priority,
long requestTimeMs) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
@@ -146,13 +149,6 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
public ContainerRequest(TaskAttemptId attemptID,
- Resource capability, String[] hosts, String[] racks,
- Priority priority, String nodeLabelExpression) {
- this(attemptID, capability, hosts, racks, priority,
- System.currentTimeMillis(), nodeLabelExpression);
- }
-
- public ContainerRequest(TaskAttemptId attemptID,
Resource capability, String[] hosts, String[] racks,
Priority priority, long requestTimeMs,String nodeLabelExpression) {
this.attemptID = attemptID;
@@ -186,6 +182,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT,
MRJobConfig.DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT);
LOG.info("maxTaskFailuresPerNode is " + maxTaskFailuresPerNode);
+ optOutOfOversubscription = conf.getBoolean(
+ MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ MRJobConfig.DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT);
+ LOG.info("optOutOfOversubscription is " + optOutOfOversubscription);
if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) {
throw new YarnRuntimeException("Invalid blacklistDisablePercent: "
+ blacklistDisablePercent
@@ -398,20 +398,20 @@ public abstract class RMContainerRequestor extends RMCommunicator {
for (String host : req.hosts) {
// Data-local
if (!isNodeBlacklisted(host)) {
- addResourceRequest(req.priority, host, req.capability,
+ addGuaranteedResourceRequest(req.priority, host, req.capability,
null);
}
}
// Nothing Rack-local for now
for (String rack : req.racks) {
- addResourceRequest(req.priority, rack, req.capability,
+ addGuaranteedResourceRequest(req.priority, rack, req.capability,
null);
}
// Off-switch
- addResourceRequest(req.priority, ResourceRequest.ANY, req.capability,
- req.nodeLabelExpression);
+ addGuaranteedResourceRequest(req.priority, ResourceRequest.ANY,
+ req.capability, req.nodeLabelExpression);
}
protected void decContainerReq(ContainerRequest req) {
@@ -430,18 +430,18 @@ public abstract class RMContainerRequestor extends RMCommunicator {
protected void addOpportunisticResourceRequest(Priority priority,
Resource capability) {
addResourceRequest(priority, ResourceRequest.ANY, capability, null,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, true);
}
- private void addResourceRequest(Priority priority, String resourceName,
- Resource capability, String nodeLabelExpression) {
+ private void addGuaranteedResourceRequest(Priority priority,
+ String resourceName, Resource capability, String nodeLabelExpression) {
addResourceRequest(priority, resourceName, capability, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, optOutOfOversubscription);
}
private void addResourceRequest(Priority priority, String resourceName,
Resource capability, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, boolean enforceExecutionType) {
Map<String, Map<Resource, ResourceRequest>> remoteRequests =
this.remoteRequestsTable.get(priority);
if (remoteRequests == null) {
@@ -464,8 +464,8 @@ public abstract class RMContainerRequestor extends RMCommunicator {
remoteRequest.setCapability(capability);
remoteRequest.setNumContainers(0);
remoteRequest.setNodeLabelExpression(nodeLabelExpression);
- remoteRequest.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(executionType, true));
+ remoteRequest.setExecutionTypeRequest(ExecutionTypeRequest.
+ newInstance(executionType, enforceExecutionType));
reqMap.put(capability, remoteRequest);
}
remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
@@ -473,9 +473,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
- LOG.debug("addResourceRequest:" + " applicationId="
+ LOG.debug("addGuaranteedResourceRequest:" + " applicationId="
+ applicationId.getId() + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
+ + " resourceName=" + resourceName + " ExecutionType=" + executionType
+ + " enforceExecutionType=" + enforceExecutionType + " numContainers="
+ remoteRequest.getNumContainers() + " #asks=" + ask.size());
}
}
@@ -559,8 +560,9 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
}
String[] hosts = newHosts.toArray(new String[newHosts.size()]);
- ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability,
- hosts, orig.racks, orig.priority, orig.nodeLabelExpression);
+ ContainerRequest newReq = new ContainerRequest(orig.attemptID,
+ orig.capability, hosts, orig.racks, orig.priority,
+ System.currentTimeMillis(), orig.nodeLabelExpression);
return newReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f00f7ad9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 427e6ea..5d4dcf0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -105,6 +105,8 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -868,6 +870,196 @@ public class TestRMContainerAllocator {
allocator.close();
}
+ /**
+ * Test A MapReduce job can be configured to opt out of oversubscription,
+ * that is, it always wait for guaranteed resources to execute its tasks.
+ * This is done by setting up a MapReduce job with 2 mappers and 1 reducers
+ * and capturing all ResourceRequests sent from the AM to RM, then checking
+ * if all ResourceRequests are guaranteed and their enforceExecutionType is
+ * true.
+ */
+ @Test
+ public void testMapReduceOptingOutOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(true);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or not enforced.");
+ }
+ }
+ }
+
+ /**
+ * Test a MapReduce job can be configured to opt in oversubscription (
+ * true by default). This is done by setting up a MapReduce job with 2
+ * mappers and 1 reducers and capturing all ResourceRequests sent from
+ * the AM to RM, then checking if all ResourceRequests are guaranteed
+ * but their enforceExecutionType is always set to false.
+ */
+ @Test
+ public void testMapReduceOptingInOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(false);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, false))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or it is enforced.");
+ }
+ }
+ }
+
+ /**
+ * Set up a mapreduce job with 2 mappers and 1 reducer and return
+ * all ResourceRequests sent from the AM to RM.
+ */
+ private List<ResourceRequest> captureResourceRequests(
+ boolean optOutOfOversubscription) throws Exception {
+ List<ResourceRequest> resourceRequests = new ArrayList<>();
+
+ final Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ optOutOfOversubscription);
+
+ // start the resource manager
+ final MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // submit an application
+ RMApp rmApp = rm.submitApp(1024);
+ rm.drainEvents();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 11264);
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ rm.drainEvents();
+
+ // start the MR AM and wait until it is in running state
+ MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(
+ appAttemptId, 0), 2, 1, false,
+ this.getClass().getName(), true, 1) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new MyContainerAllocator(rm, appAttemptId, context);
+ };
+ };
+ mrApp.submit(conf);
+
+ Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next()
+ .getValue();
+ DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
+ MyContainerAllocator allocator = (MyContainerAllocator) mrApp
+ .getContainerAllocator();
+ mrApp.waitForInternalState((JobImpl)job, JobStateInternal.RUNNING);
+ amDispatcher.await();
+
+ // wait until all attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values()
+ .iterator().next(), TaskAttemptStateInternal.UNASSIGNED);
+ }
+ amDispatcher.await();
+
+ // send map resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for both map tasks to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.MAP) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish both map tasks so that the reduce task can be scheduled
+ Iterator<Task> it = job.getTasks().values().iterator();
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // send the reduce resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for the reduce task to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.REDUCE) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish the reduce task
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ return resourceRequests;
+ }
+
@Test
public void testMapReduceScheduling() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f00f7ad9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index ca18bfe..48887c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1179,6 +1179,12 @@ public interface MRJobConfig {
public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
/**
+ * Opt out of YARN oversubscription so that the job always waits for
+ * GUARANTEED resources available in the cluster.
+ */
+ String MR_OVERSUBSCRIPTION_OPT_OUT = "mapreduce.job.oversubscription-opt-out";
+ boolean DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT = false;
+ /**
* A comma-separated list of properties whose value will be redacted.
*/
String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f00f7ad9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 9f33d65..de725a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -2151,4 +2151,12 @@
</description>
</property>
+<property>
+ <description>
+ Opts out of YARN oversubscription so that the job always waits for GUARANTEED
+ resources available.
+ </description>
+ <name>mapreduce.job.oversubscription-opt-out</name>
+ <value>false</value>
+</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-6794. Fair Scheduler to
explicitly promote OPPORTUNISITIC containers locally at the node where
they're running. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6794. Fair Scheduler to explicitly promote OPPORTUNISITIC containers locally at the node where they're running. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ff8d3f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ff8d3f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ff8d3f9
Branch: refs/heads/YARN-1011
Commit: 3ff8d3f991aeb20c9c9bc4326c7169faeec03e7f
Parents: faa71db
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jun 20 10:53:20 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:42:51 2018 -0700
----------------------------------------------------------------------
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../scheduler/SchedulerNode.java | 27 +
.../scheduler/fair/FSAppAttempt.java | 32 +-
.../scheduler/fair/FSSchedulerNode.java | 57 +-
.../scheduler/fair/FairScheduler.java | 47 +-
.../TestResourceTrackerService.java | 3 +-
.../scheduler/fair/TestFairScheduler.java | 569 +++++++++++++++++++
7 files changed, 715 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 25a9415..d812e57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -672,7 +672,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
public Resource getCurrentConsumption() {
- return attemptResourceUsage.getUsed();
+ return Resources.add(attemptResourceUsage.getUsed(),
+ attemptOpportunisticResourceUsage.getUsed());
}
private Container updateContainerAndNMToken(RMContainer rmContainer,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 3e16dc2..9e38d52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -282,6 +282,33 @@ public abstract class SchedulerNode {
return true;
}
+ /**
+ * Attempt to promote an OPPORTUNISTIC container that has been allocated.
+ * @param rmContainer the OPPORTUNISTIC container to promote
+ * @return true if the given OPPORTUNISTIC container is promoted,
+ * false otherwise
+ */
+ public synchronized boolean tryToPromoteOpportunisticContainer(
+ RMContainer rmContainer) {
+ assert (rmContainer.getExecutionType() == ExecutionType.OPPORTUNISTIC);
+
+ boolean promoted = false;
+ Resource resource = rmContainer.getContainer().getResource();
+ if (allocatedContainers.containsKey(rmContainer.getContainerId()) &&
+ Resources.fitsIn(resource, getUnallocatedResource())) {
+ Resources.subtractFrom(allocatedResourceOpportunistic, resource);
+ numOpportunisticContainers--;
+
+ Resources.addTo(allocatedResourceGuaranteed, resource);
+ numGuaranteedContainers++;
+ Resources.subtractFrom(unallocatedResource, resource);
+
+ promoted = true;
+ }
+
+ return promoted;
+ }
+
/**
* Get resources that are not allocated to GUARANTEED containers on the node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 7b32a9d..b68c245 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -505,6 +505,29 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
/**
+ * Update resource accounting upon promotion of an OPPORTUNISTIC container.
+ * @param rmContainer the OPPORTUNISTIC container that has been promoted
+ */
+ public void opportunisticContainerPromoted(RMContainer rmContainer) {
+ // only an OPPORTUNISTIC container can be promoted
+ assert (ExecutionType.OPPORTUNISTIC == rmContainer.getExecutionType());
+
+ // the container to be promoted must belong to the current app attempt
+ if (rmContainer.getApplicationAttemptId().equals(
+ getApplicationAttemptId())) {
+ Resource resource = rmContainer.getContainer().getResource();
+ try {
+ writeLock.lock();
+ attemptOpportunisticResourceUsage.decUsed(resource);
+ attemptResourceUsage.incUsed(resource);
+ getQueue().incUsedGuaranteedResource(resource);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+
+ /**
* Should be called when the scheduler assigns a container at a higher
* degree of locality than the current threshold. Reset the allowed locality
* level to a higher degree of locality.
@@ -1159,7 +1182,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*
* @param node
* Node that the application has an existing reservation on
- * @return whether the reservation on the given node is valid.
+ * @return true if the reservation is turned into an allocation
*/
boolean assignReservedContainer(FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
@@ -1186,8 +1209,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
node.getUnallocatedResource())) {
assignContainer(node, false, true);
+ return true;
}
- return true;
+ return false;
}
/**
@@ -1355,12 +1379,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
@Override
public Resource getGuaranteedResourceUsage() {
- return getCurrentConsumption();
+ return Resources.clone(attemptResourceUsage.getUsed());
}
@Override
public Resource getOpportunisticResourceUsage() {
- return attemptOpportunisticResourceUsage.getUsed();
+ return Resources.clone(attemptOpportunisticResourceUsage.getUsed());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index a53dda4..efbe615 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -35,10 +36,13 @@ import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.util.resource.Resources;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;
@@ -66,6 +70,13 @@ public class FSSchedulerNode extends SchedulerNode {
// slated for preemption
private Resource totalResourcesPreempted = Resource.newInstance(0, 0);
+ // The set of containers that need to be handled before resource
+ // available on the node can be assigned to resource requests.
+ // This is a queue of reserved and opportunistic containers on
+ // the node.
+ private final LinkedHashSet<RMContainer> priorityContainers =
+ new LinkedHashSet(1);
+
@VisibleForTesting
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
@@ -124,6 +135,7 @@ public class FSSchedulerNode extends SchedulerNode {
+ application.getApplicationId());
}
setReservedContainer(container);
+ priorityContainers.add(container);
this.reservedAppSchedulable = (FSAppAttempt) application;
}
@@ -142,7 +154,7 @@ public class FSSchedulerNode extends SchedulerNode {
" for application " + reservedApplication.getApplicationId() +
" on node " + this);
}
-
+ priorityContainers.remove(getReservedContainer());
setReservedContainer(null);
this.reservedAppSchedulable = null;
}
@@ -274,6 +286,13 @@ public class FSSchedulerNode extends SchedulerNode {
} else {
LOG.error("Allocated empty container" + rmContainer.getContainerId());
}
+
+ // keep track of opportunistic containers allocated so that we can promote
+ // them before we assign resources available to resource requests.
+ if (ExecutionType.OPPORTUNISTIC.equals(
+ rmContainer.getContainer().getExecutionType())) {
+ priorityContainers.add(rmContainer);
+ }
}
/**
@@ -292,4 +311,40 @@ public class FSSchedulerNode extends SchedulerNode {
containersForPreemption.remove(container);
}
}
+
+ /**
+ * Try to assign resources available to reserved container and opportunistic
+ * containers that have been allocated.
+ * @return the list of opportunistic containers that have been promoted
+ */
+ public synchronized List<RMContainer> handlePriorityContainers() {
+ boolean assigned = true;
+ List<RMContainer> promotedContainers = new ArrayList<>(0);
+
+ List<RMContainer> candidateContainers = new ArrayList<>(priorityContainers);
+ for (RMContainer rmContainer : candidateContainers) {
+ boolean isReservedContainer =
+ rmContainer.getReservedSchedulerKey() != null;
+ if (isReservedContainer) {
+ // attempt to assign resources that have been reserved
+ FSAppAttempt reservedApp = getReservedAppSchedulable();
+ if (reservedApp != null) {
+ reservedApp.assignReservedContainer(this);
+ }
+ } else {
+ if (super.tryToPromoteOpportunisticContainer(rmContainer)) {
+ priorityContainers.remove(rmContainer);
+ assigned = true;
+ promotedContainers.add(rmContainer);
+ }
+ }
+
+ if (!assigned) {
+ // break out of the loop because assigned being false indicates
+ // there is no more resources that are available for promotion.
+ break;
+ }
+ }
+ return promotedContainers;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 25782a1..31f1961 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -70,6 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeUpdateContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
@@ -104,6 +106,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -1113,8 +1116,8 @@ public class FairScheduler extends
// Assign new containers...
// 1. Ensure containers are assigned to the apps that preempted
- // 2. Check for reserved applications
- // 3. Schedule GUARANTEED containers if there are no reservations
+ // 2. Check for reserved applications or promote OPPORTUNISTIC containers
+ // 3. Schedule GUARANTEED containers
// 4. Schedule OPPORTUNISTIC containers if possible
// Apps may wait for preempted containers
@@ -1123,12 +1126,14 @@ public class FairScheduler extends
// when C does not qualify for preemption itself.
attemptToAssignPreemptedResources(node);
- boolean validReservation = attemptToAssignReservedResources(node);
- if (!validReservation) {
- // only attempt to assign GUARANTEED containers if there is no
- // reservation on the node because
- attemptToAssignResourcesAsGuaranteedContainers(node);
- }
+ // before we assign resources to outstanding resource requests, we
+ // need to assign the resources to either the container that has
+ // made a reservation or allocated OPPORTUNISTIC containers so that
+ // they can be promoted. This ensures that request requests that
+ // are eligible for guaranteed resources are satisfied in FIFO order
+ attemptToAssignReservedResourcesOrPromoteOpportunisticContainers(node);
+
+ attemptToAssignResourcesAsGuaranteedContainers(node);
// attempt to assign OPPORTUNISTIC containers regardless of whether
// we have made a reservation or assigned a GUARANTEED container
@@ -1143,15 +1148,27 @@ public class FairScheduler extends
}
/**
- * Assign the reserved resource to the application that have reserved it.
+ * Attempt to assign reserved resources and promote OPPORTUNISTIC containers
+ * thata have already been allocated.
*/
- private boolean attemptToAssignReservedResources(FSSchedulerNode node) {
- boolean success = false;
- FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
- if (reservedAppSchedulable != null) {
- success = reservedAppSchedulable.assignReservedContainer(node);
+ private void attemptToAssignReservedResourcesOrPromoteOpportunisticContainers(
+ FSSchedulerNode node) {
+ Map<Container, ContainerUpdateType> promotion = new HashMap<>(0);
+
+ List<RMContainer> promoted = node.handlePriorityContainers();
+ for (RMContainer rmContainer : promoted) {
+ FSAppAttempt appAttempt = getSchedulerApp(
+ rmContainer.getApplicationAttemptId());
+ appAttempt.opportunisticContainerPromoted(rmContainer);
+
+ promotion.put(rmContainer.getContainer(),
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE);
+ }
+
+ if (!promotion.isEmpty()) {
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeUpdateContainerEvent(node.getNodeID(), promotion));
}
- return success;
}
private void attemptToAssignResourcesAsGuaranteedContainers(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index e40b3c0..360d1af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2309,9 +2309,8 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rc.getExecutionType());
}
- // Should only include GUARANTEED resources
currentConsumption = applicationAttempt.getCurrentConsumption();
- Assert.assertEquals(Resource.newInstance(2048, 1), currentConsumption);
+ Assert.assertEquals(Resource.newInstance(5120, 3), currentConsumption);
allocResources =
applicationAttempt.getQueue().getMetrics().getAllocatedResources();
Assert.assertEquals(Resource.newInstance(2048, 1), allocResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ff8d3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 5878ccd..7fbf84a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3353,6 +3353,575 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test promotion of a single OPPORTUNISTIC container when no resources are
+ * reserved on the node where the container is allocated.
+ */
+ @Test
+ public void testSingleOpportunisticContainerPromotionWithoutReservation()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("No reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test promotion of two OPPORTUNISTIC containers when no resources are
+ * reserved on the node where the container is allocated.
+ */
+ @Test
+ public void testMultipleOpportunisticContainerPromotionWithoutReservation()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1536, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1536, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("No reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ // node utilization is low after the third container run on the node
+ ContainerStatus container3Status = ContainerStatus.newInstance(
+ allocatedContainers3.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(container3Status),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(2000, 0, 0.2f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt4 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers4.get(0).getExecutionType());
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ // the first OPPORTUNISTIC container should be promoted
+ assertEquals(1536, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ // the second OPPORLTUNISTIC container should not be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+
+ // now the second GUARANTEED container finishes
+ finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers2.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(3000, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ // the second OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test promotion of OPPORTUNISTIC container when there is resources
+ * reserved before the container is allocated. The scheduler should
+ * satisfy the reservation first before it promotes the OPPORTUNISTIC
+ * container when resources are released.
+ */
+ @Test
+ public void testOpportunisticContainerPromotionWithPriorReservation()
+ throws Exception {
+
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that opts out of oversubscription
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(2000, "queue2", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt4 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers4.get(0).getExecutionType());
+ assertTrue("A reservation should still be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the reserved container of the third request that opted out of
+ // oversubscription should now be satisfied with a GUARANTEED container
+ assertEquals(2000, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("The reservation for the third request should be canceled",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ // the OPPORTUNISTIC container should not be promoted given the released
+ // resources are taken by handling the reservation
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // now the second GUARANTEED container finishes
+ finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers2.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(3000, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test promotion of OPPORTUNISTIC container when there is resources
+ * reserved after the container is allocated. The scheduler should
+ * promotes the OPPORTUNISTIC container before it satisfy the reservation
+ * when resources are released.
+ */
+ @Test
+ public void testOpportunisticContainerPromotionWithPostReservation()
+ throws Exception {
+
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("No reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ // create another scheduling request that opts out of oversubscription
+ ApplicationAttemptId appAttempt4 =
+ createSchedulingRequest(2000, "queue3", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 0);
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the fourth request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertTrue("A reservation should still be made for the fourth request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // now the second GUARANTEED container finishes
+ finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers2.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(3000, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the reserved container of the fourth request that opted out of
+ // oversubscription should now be satisfied with a GUARANTEED container
+ assertEquals(2000, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers4.get(0).getExecutionType());
+ assertTrue("The reservation for the fourth request should be canceled",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-15684. triggerActiveLogRoll
stuck on dead name node,
when ConnectTimeoutException happens. Contributed by Rong Tang.
Posted by ha...@apache.org.
HADOOP-15684. triggerActiveLogRoll stuck on dead name node, when ConnectTimeoutException happens. Contributed by Rong Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/090272d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/090272d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/090272d7
Branch: refs/heads/YARN-1011
Commit: 090272d7de7cb5f1133359d66780aef7c5cce5c9
Parents: 236d16e
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Sep 19 12:58:31 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Sep 19 13:00:30 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/ha/EditLogTailer.java | 34 ++-----
.../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 ++++
.../server/namenode/ha/TestEditLogTailer.java | 98 +++++++++++++-------
3 files changed, 93 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 4ba2aa3..fc5f3a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -53,8 +53,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.SecurityUtil;
import static org.apache.hadoop.util.Time.monotonicNow;
@@ -382,15 +380,6 @@ public class EditLogTailer {
future.get(rollEditsTimeoutMs, TimeUnit.MILLISECONDS);
lastRollTriggerTxId = lastLoadedTxnId;
} catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof RemoteException) {
- IOException ioe = ((RemoteException) cause).unwrapRemoteException();
- if (ioe instanceof StandbyException) {
- LOG.info("Skipping log roll. Remote node is not in Active state: " +
- ioe.getMessage().split("\n")[0]);
- return;
- }
- }
LOG.warn("Unable to trigger a roll of the active NN", e);
} catch (TimeoutException e) {
if (future != null) {
@@ -497,7 +486,8 @@ public class EditLogTailer {
* This mechanism is <b>very bad</b> for cases where we care about being <i>fast</i>; it just
* blindly goes and tries namenodes.
*/
- private abstract class MultipleNameNodeProxy<T> implements Callable<T> {
+ @VisibleForTesting
+ abstract class MultipleNameNodeProxy<T> implements Callable<T> {
/**
* Do the actual work to the remote namenode via the {@link #cachedActiveProxy}.
@@ -513,19 +503,13 @@ public class EditLogTailer {
try {
T ret = doWork();
return ret;
- } catch (RemoteException e) {
- Throwable cause = e.unwrapRemoteException(StandbyException.class);
- // if its not a standby exception, then we need to re-throw it, something bad has happened
- if (cause == e) {
- throw e;
- } else {
- // it is a standby exception, so we try the other NN
- LOG.warn("Failed to reach remote node: " + currentNN
- + ", retrying with remaining remote NNs");
- cachedActiveProxy = null;
- // this NN isn't responding to requests, try the next one
- nnLoopCount++;
- }
+ } catch (IOException e) {
+ LOG.warn("Exception from remote name node " + currentNN
+ + ", try next.", e);
+
+ // Try next name node if exception happens.
+ cachedActiveProxy = null;
+ nnLoopCount++;
}
}
throw new IOException("Cannot find any valid remote NN to service request!");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
index c21ff80..390c61c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
@@ -73,6 +73,23 @@ public class MiniDFSNNTopology {
}
/**
+ * Set up an HA topology with a single HA nameservice.
+ * @param nnCount of namenodes to use with the nameservice
+ * @param basePort for IPC and Http ports of namenodes.
+ */
+ public static MiniDFSNNTopology simpleHATopology(int nnCount, int basePort) {
+ MiniDFSNNTopology.NSConf ns = new MiniDFSNNTopology.NSConf("minidfs-ns");
+ for (int i = 0; i < nnCount; i++) {
+ ns.addNN(new MiniDFSNNTopology.NNConf("nn" + i)
+ .setIpcPort(basePort++)
+ .setHttpPort(basePort++));
+ }
+ MiniDFSNNTopology topology = new MiniDFSNNTopology()
+ .addNameservice(ns);
+ return topology;
+ }
+
+ /**
* Set up federated cluster with the given number of nameservices, each
* of which has only a single NameNode.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
index 68b3e2b..b94cd2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
@@ -28,6 +28,7 @@ import java.net.BindException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -46,7 +47,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.event.Level;
import org.junit.Test;
@@ -177,21 +177,7 @@ public class TestEditLogTailer {
MiniDFSCluster cluster = null;
for (int i = 0; i < 5; i++) {
try {
- // Have to specify IPC ports so the NNs can talk to each other.
- int[] ports = ServerSocketUtil.getPorts(3);
- MiniDFSNNTopology topology = new MiniDFSNNTopology()
- .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1")
- .setIpcPort(ports[0]))
- .addNN(new MiniDFSNNTopology.NNConf("nn2")
- .setIpcPort(ports[1]))
- .addNN(new MiniDFSNNTopology.NNConf("nn3")
- .setIpcPort(ports[2])));
-
- cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .build();
+ cluster = createMiniDFSCluster(conf, 3);
break;
} catch (BindException e) {
// retry if race on ports given by ServerSocketUtil#getPorts
@@ -222,21 +208,9 @@ public class TestEditLogTailer {
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
- // Have to specify IPC ports so the NNs can talk to each other.
- MiniDFSNNTopology topology = new MiniDFSNNTopology()
- .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1")
- .setIpcPort(ServerSocketUtil.getPort(0, 100)))
- .addNN(new MiniDFSNNTopology.NNConf("nn2")
- .setIpcPort(ServerSocketUtil.getPort(0, 100)))
- .addNN(new MiniDFSNNTopology.NNConf("nn3")
- .setIpcPort(ServerSocketUtil.getPort(0, 100))));
-
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .build();
+ MiniDFSCluster cluster = null;
try {
+ cluster = createMiniDFSCluster(conf, 3);
cluster.transitionToStandby(0);
cluster.transitionToStandby(1);
cluster.transitionToStandby(2);
@@ -249,7 +223,9 @@ public class TestEditLogTailer {
cluster.transitionToActive(0);
waitForLogRollInSharedDir(cluster, 3);
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -316,4 +292,64 @@ public class TestEditLogTailer {
cluster.shutdown();
}
}
+
+ @Test
+ public void testRollEditLogIOExceptionForRemoteNN() throws IOException {
+ Configuration conf = getConf();
+
+ // Roll every 1s
+ conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = createMiniDFSCluster(conf, 3);
+ cluster.transitionToActive(0);
+ EditLogTailer tailer = Mockito.spy(
+ cluster.getNamesystem(1).getEditLogTailer());
+
+ final AtomicInteger invokedTimes = new AtomicInteger(0);
+
+ // It should go on to next name node when IOException happens.
+ when(tailer.getNameNodeProxy()).thenReturn(
+ tailer.new MultipleNameNodeProxy<Void>() {
+ @Override
+ protected Void doWork() throws IOException {
+ invokedTimes.getAndIncrement();
+ throw new IOException("It is an IO Exception.");
+ }
+ }
+ );
+
+ tailer.triggerActiveLogRoll();
+
+ // MultipleNameNodeProxy uses Round-robin to look for active NN
+ // to do RollEditLog. If doWork() fails, then IOException throws,
+ // it continues to try next NN. triggerActiveLogRoll finishes
+ // either due to success, or using up retries.
+ // In this test case, there are 2 remote name nodes, default retry is 3.
+ // For test purpose, doWork() always returns IOException,
+ // so the total invoked times will be default retry 3 * remote NNs 2 = 6
+ assertEquals(6, invokedTimes.get());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private static MiniDFSCluster createMiniDFSCluster(Configuration conf,
+ int nnCount) throws IOException {
+ int basePort = 10060 + new Random().nextInt(100) * 2;
+
+ // By passing in basePort, name node will have IPC port set,
+ // which is needed for enabling roll log.
+ MiniDFSNNTopology topology =
+ MiniDFSNNTopology.simpleHATopology(nnCount, basePort);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology)
+ .numDataNodes(0)
+ .build();
+ return cluster;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-1015. FS should watch node
resource utilization and allocate opportunistic containers if appropriate.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 4b637ff..ff620b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -57,8 +57,13 @@ import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -66,6 +71,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -77,6 +83,8 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -99,6 +107,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -1062,15 +1071,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(updateEvent2);
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(2, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getVirtualCores());
+ getGuaranteedResourceUsage().getVirtualCores());
// verify metrics
QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1")
@@ -1105,7 +1114,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -1115,7 +1124,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemorySize());
// Now another node checks in with capacity
@@ -1129,7 +1138,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure this goes to queue 2
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// The old reservation should still be there...
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemorySize());
@@ -1139,7 +1148,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
- @Test (timeout = 5000)
+ @Test
public void testOffSwitchAppReservationThreshold() throws Exception {
conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f);
scheduler.init(conf);
@@ -1179,7 +1188,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(6144, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Create new app with a resource request that can be satisfied by any
// node but would be
@@ -1211,7 +1220,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -1272,7 +1281,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Create new app with a resource request that can be satisfied by any
// node but would be
@@ -1317,7 +1326,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
assertEquals(10240, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -1361,7 +1370,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Verify number of reservations have decremented
assertEquals(0,
@@ -1405,7 +1414,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
createSchedulingRequest(1024, "queue2", "user2", 1);
@@ -1414,7 +1423,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
@@ -1540,7 +1549,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
createSchedulingRequest(1024, "queue2", "user2", 1);
@@ -1549,7 +1558,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
@@ -1589,12 +1598,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure allocated memory of queue1 doesn't exceed its maximum
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
//the reservation of queue1 should be reclaim
assertEquals(0, scheduler.getSchedulerApp(attId1).
getCurrentReservation().getMemorySize());
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
}
@Test
@@ -1634,7 +1643,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests below threshold
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -1643,7 +1652,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 has no reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(0,
scheduler.getSchedulerApp(attId).getReservedContainers().size());
@@ -1654,7 +1663,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
.getVirtualCores());
@@ -1669,7 +1678,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure this goes to queue 2
assertEquals(3, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getVirtualCores());
+ getGuaranteedResourceUsage().getVirtualCores());
// The old reservation should still be there...
assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
@@ -2702,7 +2711,361 @@ public class TestFairScheduler extends FairSchedulerTestBase {
2, liveContainers.iterator().next().getContainer().
getPriority().getPriority());
}
-
+
+ /**
+ * Test that NO OPPORTUNISTIC containers can be allocated on a node that
+ * is fully allocated and with a very high utilization.
+ */
+ @Test
+ public void testAllocateNoOpportunisticContainersOnBusyNode()
+ throws IOException {
+ conf.setBoolean(
+ YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED, true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(2048, 2), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the node's full memory
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization shoots up after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(2000, 0, 0.8f));
+
+ // create another scheduling request
+ ApplicationAttemptId appAttempt2
+ = createSchedulingRequest(100, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue("Expecting no containers allocated",
+ allocatedContainers2.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // verify that a reservation is made for the second resource request
+ Resource reserved = scheduler.getNode(node.getNodeID()).
+ getReservedContainer().getReservedResource();
+ assertTrue("Expect a reservation made for the second resource request",
+ reserved.equals(Resource.newInstance(100, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test that OPPORTUNISTIC containers can be allocated on a node with low
+ * utilization even though there is not enough unallocated resource on the
+ * node to accommodate the request.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersOnPartiallyOverAllocatedNode()
+ throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1800, 0, 0.5f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // verify that no reservation is made for the second request given
+ // that it's satisfied by an OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made because we have satisfied" +
+ " the second request with an OPPORTUNISTIC container allocation",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test opportunistic containers can be allocated on a node that is fully
+ * allocated but whose utilization is very low.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersOnFullyAllocatedNode()
+ throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 = createSchedulingRequest(
+ 4096, "queue1", "user1", 4);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1800, 0, 0.5f));
+
+ // create another scheduling request now that there is no unallocated
+ // resources left on the node, the request should be served with an
+ // allocation of an opportunistic container
+ ApplicationAttemptId appAttempt2 = createSchedulingRequest(
+ 1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // verify that no reservation is made for the second request given
+ // that it's satisfied by an OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made because we have satisfied" +
+ " the second request with an OPPORTUNISTIC container allocation",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test opportunistic containers can be allocated on a node with a low
+ * utilization even though there are GUARANTEED containers allocated.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersWithGuaranteedOnes()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3200, "queue1", "user1", 3);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3200, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(512, 0, 0.1f));
+
+ // create two other scheduling requests which in aggregate ask for more
+ // that what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(512, "queue2", "user1", 1);
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(512, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // verify that no reservation is made given that the second request should
+ // be satisfied by a GUARANTEED container allocation, the third by an
+ // OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made.",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
@@ -3692,7 +4055,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
.createAbnormalContainerStatus(container.getContainerId(),
SchedulerUtils.COMPLETED_APPLICATION),
RMContainerEventType.FINISHED);
- assertEquals(Resources.none(), app1.getResourceUsage());
+ assertEquals(Resources.none(), app1.getGuaranteedResourceUsage());
}
@Test
@@ -3792,7 +4155,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application1's AM should be finished",
0, app1.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app1.getResourceUsage());
+ Resources.none(), app1.getGuaranteedResourceUsage());
assertEquals("Application3's AM should be running",
1, app3.getLiveContainers().size());
assertEquals("Application3's AM requests 1024 MB memory",
@@ -3812,7 +4175,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application4's AM should not be running",
0, app4.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app4.getResourceUsage());
+ Resources.none(), app4.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3828,7 +4191,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3841,7 +4204,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3857,11 +4220,11 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application2's AM should be finished",
0, app2.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app2.getResourceUsage());
+ Resources.none(), app2.getGuaranteedResourceUsage());
assertEquals("Application3's AM should be finished",
0, app3.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app3.getResourceUsage());
+ Resources.none(), app3.getGuaranteedResourceUsage());
assertEquals("Application5's AM should be running",
1, app5.getLiveContainers().size());
assertEquals("Application5's AM requests 2048 MB memory",
@@ -3882,7 +4245,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should have 0 container",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
scheduler.update();
@@ -3906,7 +4269,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application6's AM should not be running",
0, app6.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app6.getResourceUsage());
+ Resources.none(), app6.getGuaranteedResourceUsage());
assertEquals("Application6's AM resource shouldn't be updated",
0, app6.getAMResource().getMemorySize());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
@@ -4621,17 +4984,25 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSQueue queue2 = queueMgr.getLeafQueue("parent2.queue2", true);
FSQueue queue1 = queueMgr.getLeafQueue("parent1.queue1", true);
- Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(parent2.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(queue2.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(parent1.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(queue1.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
scheduler.moveApplication(appAttId.getApplicationId(), "parent2.queue2");
- Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(parent2.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(queue2.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(parent1.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(queue1.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
}
@Test (expected = YarnException.class)
@@ -4671,7 +5042,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
scheduler.handle(updateEvent);
- assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(2048, 2),
+ oldQueue.getGuaranteedResourceUsage());
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
@@ -5117,7 +5489,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
//container will be reserved at node1
RMContainer reservedContainer1 =
@@ -5137,7 +5509,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
app1, RMAppAttemptState.KILLED, false));
assertEquals(0, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// container will be allocated at node2
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -5285,10 +5657,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app1 = mock(FSAppAttempt.class);
Mockito.when(app1.getDemand()).thenReturn(maxResource);
- Mockito.when(app1.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app1.getGuaranteedResourceUsage()).
+ thenReturn(Resources.none());
FSAppAttempt app2 = mock(FSAppAttempt.class);
Mockito.when(app2.getDemand()).thenReturn(maxResource);
- Mockito.when(app2.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app2.getGuaranteedResourceUsage()).
+ thenReturn(Resources.none());
QueueManager queueManager = scheduler.getQueueManager();
FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
@@ -5344,7 +5718,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
child1.setMaxShare(new ConfigurableResource(resource));
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(resource);
- Mockito.when(app.getResourceUsage()).thenReturn(resource);
+ Mockito.when(app.getGuaranteedResourceUsage()).thenReturn(resource);
child1.addApp(app, true);
child1.updateDemand();
@@ -5380,7 +5754,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
+ " SteadyFairShare: <memory:0, vCores:0>,"
+ " MaxShare: <memory:4096, vCores:4>,"
+ " MinShare: <memory:0, vCores:0>,"
- + " ResourceUsage: <memory:4096, vCores:4>,"
+ + " Guaranteed ResourceUsage: <memory:4096, vCores:4>,"
+ " Demand: <memory:4096, vCores:4>,"
+ " MaxAMShare: 0.5,"
+ " Runnable: 0}";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/495d43ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
index b016c1b..6777b5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
@@ -243,11 +243,16 @@ public class TestSchedulingPolicy {
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return usage;
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
public Resource getMinShare() {
return minShare;
}
@@ -278,7 +283,8 @@ public class TestSchedulingPolicy {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node,
+ boolean opportunistic) {
throw new UnsupportedOperationException();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-6705 Add separate NM
preemption thresholds for cpu and memory (Haibo Chen)
Posted by ha...@apache.org.
YARN-6705 Add separate NM preemption thresholds for cpu and memory (Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7cdf2ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7cdf2ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7cdf2ce
Branch: refs/heads/YARN-1011
Commit: d7cdf2cecc128b69c15aac87e60f77ee7e462a2e
Parents: 013c977
Author: Haibo Chen <ha...@apache.org>
Authored: Wed Jul 12 12:32:13 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 14:10:13 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 31 +++++++++++++--
.../src/main/resources/yarn-default.xml | 34 ++++++++++++++--
.../monitor/ContainersMonitorImpl.java | 42 +++++++++++++-------
3 files changed, 85 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7cdf2ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index daf9910..18ff086 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2144,10 +2144,33 @@ public class YarnConfiguration extends Configuration {
public static final String NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD =
NM_PREFIX + "overallocation.memory-utilization-threshold";
- public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD =
- NM_PREFIX + "overallocation.preemption-threshold";
- public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD
- = 0.96f;
+ /**
+ * The CPU utilization threshold, if went beyond for a few times in a row,
+ * OPPORTUNISTIC containers started due to overallocation should start
+ * getting preempted.
+ */
+ public static final String NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold.cpu";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD = 0.99f;
+
+ /**
+ * The number of times that CPU utilization must go over the CPU preemption
+ * threshold consecutively before preemption starts to kick in.
+ */
+ public static final String NM_OVERALLOCATION_PREEMPTION_CPU_COUNT =
+ NM_PREFIX + "overallocation.preemption-threshold-count.cpu";
+ public static final int DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT = 4;
+
+
+ /**
+ * The memory utilization threshold beyond which OPPORTUNISTIC containers
+ * started due to overallocation should start getting preempted.
+ */
+ public static final String NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD =
+ NM_PREFIX + "overallocation.preemption-threshold.memory";
+ public static final float
+ DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD = 0.95f;
/**
* Interval of time the linux container executor should try cleaning up
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7cdf2ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 84a4225..985936a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1769,11 +1769,37 @@
<property>
<description>When a node is over-allocated to improve utilization by
- running OPPORTUNISTIC containers, this config captures the utilization
- beyond which OPPORTUNISTIC containers should start getting preempted.
+ running OPPORTUNISTIC containers, this config captures the CPU
+ utilization beyond which OPPORTUNISTIC containers should start getting
+ preempted. This is used in combination with
+ yarn.nodemanager.overallocation.preemption-threshold-count.cpu, that is,
+ only when the CPU utilization goes over this threshold consecutively for
+ a few times will preemption kicks in.
</description>
- <name>yarn.nodemanager.overallocation.preemption-threshold</name>
- <value>0.96</value>
+ <name>yarn.nodemanager.overallocation.preemption-threshold.cpu</name>
+ <value>0.99</value>
+ </property>
+
+ <property>
+ <description>When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the number of
+ times that CPU utilization has to go above
+ ${yarn.nodemanager.overallocation.preemption-threshold.cpu}
+ consecutively for NM to start preempting OPPORTUNISTIC containers
+ started due to overallocation.
+ </description>
+ <name>yarn.nodemanager.overallocation.preemption-threshold-count.cpu</name>
+ <value>4</value>
+ </property>
+
+ <property>
+ <description>When a node is over-allocated to improve utilization by
+ running OPPORTUNISTIC containers, this config captures the CPU
+ utilization beyond which OPPORTUNISTIC containers should start getting
+ preempted.
+ </description>
+ <name>yarn.nodemanager.overallocation.preemption-threshold.memory</name>
+ <value>0.95</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7cdf2ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index d7c81bb..70a8ebf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -114,6 +114,7 @@ public class ContainersMonitorImpl extends AbstractService implements
private ResourceUtilization containersUtilization;
private ResourceThresholds overAllocationPreemptionThresholds;
+ private int overAlloctionPreemptionCpuCount = -1;
private volatile boolean stopped = false;
@@ -289,7 +290,7 @@ public class ContainersMonitorImpl extends AbstractService implements
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
if (overAllocationMemoryUtilizationThreshold <= 0) {
LOG.info("NodeManager oversubscription is disabled because the memory " +
- "utilization threshold is no larger than zero.");
+ "overallocation threshold is no larger than zero.");
return;
}
@@ -301,36 +302,49 @@ public class ContainersMonitorImpl extends AbstractService implements
YarnConfiguration.MAX_NM_OVERALLOCATION_THRESHOLD);
if (overAllocationCpuUtilizationThreshold <= 0) {
LOG.info("NodeManager oversubscription is disabled because the CPU " +
- "utilization threshold is no larger than zero.");
+ "overallocation threshold is no larger than zero.");
return;
}
- float preemptionThreshold = conf.getFloat(
- YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD,
- YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD);
- if (preemptionThreshold <= overAllocationCpuUtilizationThreshold) {
- LOG.info("NodeManager oversubscription is disabled because preemption" +
- "threshold is no larger than the cpu utilization threshold.");
+ float cpuPreemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
+ YarnConfiguration.
+ DEFAULT_NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD);
+ if (cpuPreemptionThreshold <= overAllocationCpuUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because the cpu " +
+ " preemption threshold is no larger than the cpu overallocation" +
+ " threshold.");
return;
}
- if (preemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
- LOG.info("NodeManager oversubscription is disabled because preemption" +
- "threshold is no larger than the memory utilization threshold.");
+
+ float memoryPreemptionThreshold = conf.getFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD,
+ YarnConfiguration.
+ DEFAULT_NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD);
+ if (memoryPreemptionThreshold <= overAllocationMemoryUtilizationThreshold) {
+ LOG.info("NodeManager oversubscription is disabled because the memory" +
+ " preemption threshold is no larger than the memory overallocation" +
+ " threshold.");
return;
}
+ this.overAlloctionPreemptionCpuCount = conf.getInt(
+ YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT,
+ YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_CPU_COUNT);
+
ResourceThresholds resourceThresholds = ResourceThresholds.newInstance(
overAllocationCpuUtilizationThreshold,
overAllocationMemoryUtilizationThreshold);
((NodeManager.NMContext) context).setOverAllocationInfo(
OverAllocationInfo.newInstance(resourceThresholds));
- this.overAllocationPreemptionThresholds =
- ResourceThresholds.newInstance(preemptionThreshold);
+ this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
+ cpuPreemptionThreshold, memoryPreemptionThreshold);
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
- " threshold: " + preemptionThreshold);
+ " threshold (memory:" + memoryPreemptionThreshold + ", CPU:" +
+ cpuPreemptionThreshold + ")");
}
private boolean isResourceCalculatorAvailable() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-15764. [JDK10] Migrate from
sun.net.dns.ResolverConfiguration to the replacement. Contributed by Akira
Ajisaka.
Posted by ha...@apache.org.
HADOOP-15764. [JDK10] Migrate from sun.net.dns.ResolverConfiguration to the replacement. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/429a07e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/429a07e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/429a07e0
Branch: refs/heads/YARN-1011
Commit: 429a07e08c8c919b1679c0a80df73d147d95e8a6
Parents: 3da94a3
Author: Ewan Higgs <ew...@wdc.com>
Authored: Thu Sep 20 15:13:55 2018 +0200
Committer: Ewan Higgs <ew...@wdc.com>
Committed: Thu Sep 20 15:13:55 2018 +0200
----------------------------------------------------------------------
.../hadoop-client-minicluster/pom.xml | 17 ++++-------------
.../hadoop-client-runtime/pom.xml | 11 +++++++++++
hadoop-common-project/hadoop-common/pom.xml | 5 +++++
.../org/apache/hadoop/security/SecurityUtil.java | 19 +++++++++++++------
4 files changed, 33 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/429a07e0/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index ea8d680..70fca8a 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -318,6 +318,10 @@
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- Add optional runtime dependency on the in-development timeline server module
@@ -773,19 +777,6 @@
<exclude>ehcache-core.xsd</exclude>
</excludes>
</filter>
-
- <!-- remove utility classes which are not required from
- dnsjava -->
- <filter>
- <artifact>dnsjava:dnsjava</artifact>
- <excludes>
- <excldue>dig*</excldue>
- <exclude>jnamed*</exclude>
- <exlcude>lookup*</exlcude>
- <exclude>update*</exclude>
- </excludes>
- </filter>
-
</filters>
<!-- relocate classes from mssql-jdbc -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/429a07e0/hadoop-client-modules/hadoop-client-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
index 532fae9..bfa6c15 100644
--- a/hadoop-client-modules/hadoop-client-runtime/pom.xml
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -212,6 +212,17 @@
<exclude>ccache.txt</exclude>
</excludes>
</filter>
+ <!-- remove utility classes which are not required from
+ dnsjava -->
+ <filter>
+ <artifact>dnsjava:dnsjava</artifact>
+ <excludes>
+ <exclude>dig*</exclude>
+ <exclude>jnamed*</exclude>
+ <exclude>lookup*</exclude>
+ <exclude>update*</exclude>
+ </excludes>
+ </filter>
</filters>
<relocations>
<relocation>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/429a07e0/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 695dcde..1e6da92 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -324,6 +324,11 @@
<artifactId>mockwebserver</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/429a07e0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 0de334a..9fea535 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -27,6 +27,7 @@ import java.net.URI;
import java.net.UnknownHostException;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -52,8 +53,9 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ZKUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-//this will need to be replaced someday when there is a suitable replacement
-import sun.net.dns.ResolverConfiguration;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.ResolverConfig;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.InetAddresses;
@@ -584,10 +586,15 @@ public final class SecurityUtil {
* hadoop.security.token.service.use_ip=false
*/
protected static class QualifiedHostResolver implements HostResolver {
- @SuppressWarnings("unchecked")
- private List<String> searchDomains =
- ResolverConfiguration.open().searchlist();
-
+ private List<String> searchDomains;
+ {
+ ResolverConfig resolverConfig = ResolverConfig.getCurrentConfig();
+ searchDomains = new ArrayList<>();
+ for (Name name : resolverConfig.searchPath()) {
+ searchDomains.add(name.toString());
+ }
+ }
+
/**
* Create an InetAddress with a fully qualified hostname of the given
* hostname. InetAddress does not qualify an incomplete hostname that
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-8769. [Submarine] Allow user
to specify customized quicklink(s) when submit Submarine job. Contributed by
Wangda Tan.
Posted by ha...@apache.org.
YARN-8769. [Submarine] Allow user to specify customized quicklink(s) when submit Submarine job. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cd63461
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cd63461
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cd63461
Branch: refs/heads/YARN-1011
Commit: 0cd63461021cc7cac39e7cc2bfaafd609c82fc79
Parents: a275277
Author: Sunil G <su...@apache.org>
Authored: Fri Sep 21 23:39:22 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Fri Sep 21 23:39:22 2018 +0530
----------------------------------------------------------------------
.../yarn/submarine/client/cli/CliConstants.java | 1 +
.../yarn/submarine/client/cli/RunJobCli.java | 8 ++
.../submarine/client/cli/param/Quicklink.java | 71 ++++++++++++++
.../client/cli/param/RunJobParameters.java | 18 ++++
.../yarnservice/YarnServiceJobSubmitter.java | 99 ++++++++++++++------
.../runtimes/yarnservice/YarnServiceUtils.java | 47 ++++++++--
.../yarnservice/TestYarnServiceRunJobCli.java | 94 +++++++++++++++++++
7 files changed, 303 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index d51ffc7..454ff1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -49,6 +49,7 @@ public class CliConstants {
public static final String WAIT_JOB_FINISH = "wait_job_finish";
public static final String PS_DOCKER_IMAGE = "ps_docker_image";
public static final String WORKER_DOCKER_IMAGE = "worker_docker_image";
+ public static final String QUICKLINK = "quicklink";
public static final String TENSORBOARD_DOCKER_IMAGE =
"tensorboard_docker_image";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
index faa22d3..5054a94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
@@ -117,6 +117,14 @@ public class RunJobCli extends AbstractCli {
options.addOption(CliConstants.WORKER_DOCKER_IMAGE, true,
"Specify docker image for WORKER, when this is not specified, WORKER "
+ "uses --" + CliConstants.DOCKER_IMAGE + " as default.");
+ options.addOption(CliConstants.QUICKLINK, true, "Specify quicklink so YARN"
+ + "web UI shows link to given role instance and port. When "
+ + "--tensorboard is speciied, quicklink to tensorboard instance will "
+ + "be added automatically. The format of quick link is: "
+ + "Quick_link_label=http(or https)://role-name:port. For example, "
+ + "if want to link to first worker's 7070 port, and text of quicklink "
+ + "is Notebook_UI, user need to specify --quicklink "
+ + "Notebook_UI=https://master-0:7070");
options.addOption("h", "help", false, "Print help");
return options;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/Quicklink.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/Quicklink.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/Quicklink.java
new file mode 100644
index 0000000..ea8732c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/Quicklink.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.yarn.submarine.client.cli.param;
+
+import org.apache.commons.cli.ParseException;
+
+/**
+ * A class represents quick links to a web page.
+ */
+public class Quicklink {
+ private String label;
+ private String componentInstanceName;
+ private String protocol;
+ private int port;
+
+ public void parse(String quicklinkStr) throws ParseException {
+ if (!quicklinkStr.contains("=")) {
+ throw new ParseException("Should be <label>=<link> format for quicklink");
+ }
+
+ int index = quicklinkStr.indexOf("=");
+ label = quicklinkStr.substring(0, index);
+ quicklinkStr = quicklinkStr.substring(index + 1);
+
+ if (quicklinkStr.startsWith("http://")) {
+ protocol = "http://";
+ } else if (quicklinkStr.startsWith("https://")) {
+ protocol = "https://";
+ } else {
+ throw new ParseException("Quicklink should start with http or https");
+ }
+
+ quicklinkStr = quicklinkStr.substring(protocol.length());
+ index = quicklinkStr.indexOf(":");
+
+ if (index == -1) {
+ throw new ParseException("Quicklink should be componet-id:port form");
+ }
+
+ componentInstanceName = quicklinkStr.substring(0, index);
+ port = Integer.parseInt(quicklinkStr.substring(index + 1));
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public String getComponentInstanceName() {
+ return componentInstanceName;
+ }
+
+ public String getProtocol() {
+ return protocol;
+ }
+
+ public int getPort() {
+ return port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index 4558f6a..92a1883 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.submarine.client.cli.CliUtils;
import org.apache.hadoop.yarn.submarine.common.ClientContext;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
/**
* Parameters used to run a job
@@ -41,6 +43,7 @@ public class RunJobParameters extends RunParameters {
private String tensorboardDockerImage;
private String workerLaunchCmd;
private String psLaunchCmd;
+ private List<Quicklink> quicklinks = new ArrayList<>();
private String psDockerImage = null;
private String workerDockerImage = null;
@@ -119,6 +122,17 @@ public class RunJobParameters extends RunParameters {
this.waitJobFinish = true;
}
+ // Quicklinks
+ String[] quicklinkStrs = parsedCommandLine.getOptionValues(
+ CliConstants.QUICKLINK);
+ if (quicklinkStrs != null) {
+ for (String ql : quicklinkStrs) {
+ Quicklink quicklink = new Quicklink();
+ quicklink.parse(ql);
+ quicklinks.add(quicklink);
+ }
+ }
+
psDockerImage = parsedCommandLine.getOptionValue(
CliConstants.PS_DOCKER_IMAGE);
workerDockerImage = parsedCommandLine.getOptionValue(
@@ -247,4 +261,8 @@ public class RunJobParameters extends RunParameters {
public String getTensorboardDockerImage() {
return tensorboardDockerImage;
}
+
+ public List<Quicklink> getQuicklinks() {
+ return quicklinks;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index 8fb213f..5855287 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -15,7 +15,6 @@
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -29,6 +28,7 @@ import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.api.records.ResourceInformation;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.submarine.client.cli.param.Quicklink;
import org.apache.hadoop.yarn.submarine.client.cli.param.RunJobParameters;
import org.apache.hadoop.yarn.submarine.common.ClientContext;
import org.apache.hadoop.yarn.submarine.common.Envs;
@@ -40,10 +40,14 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
@@ -54,6 +58,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
* Submit a job to cluster
*/
public class YarnServiceJobSubmitter implements JobSubmitter {
+ public static final String TENSORBOARD_QUICKLINK_LABEL = "Tensorboard";
private static final Logger LOG =
LoggerFactory.getLogger(YarnServiceJobSubmitter.class);
ClientContext clientContext;
@@ -98,7 +103,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
}
private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
- FileWriter fw, Component comp) throws IOException {
+ PrintWriter fw, Component comp) throws IOException {
// Find envs to use HDFS
String hdfsHome = null;
String javaHome = null;
@@ -191,7 +196,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
envs.put(Envs.TASK_TYPE_ENV, taskType.name());
}
- private String getUserName() {
+ @VisibleForTesting
+ protected String getUserName() {
return System.getProperty("user.name");
}
@@ -205,18 +211,19 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
private String generateCommandLaunchScript(RunJobParameters parameters,
TaskType taskType, Component comp) throws IOException {
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
- FileWriter fw = new FileWriter(file);
+ Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+ PrintWriter pw = new PrintWriter(w);
try {
- fw.append("#!/bin/bash\n");
+ pw.append("#!/bin/bash\n");
- addHdfsClassPathIfNeeded(parameters, fw, comp);
+ addHdfsClassPathIfNeeded(parameters, pw, comp);
if (taskType.equals(TaskType.TENSORBOARD)) {
String tbCommand =
"export LC_ALL=C && tensorboard --logdir=" + parameters
.getCheckpointPath();
- fw.append(tbCommand + "\n");
+ pw.append(tbCommand + "\n");
LOG.info("Tensorboard command=" + tbCommand);
} else{
// When distributed training is required
@@ -226,20 +233,20 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
taskType.getComponentName(), parameters.getNumWorkers(),
parameters.getNumPS(), parameters.getName(), getUserName(),
getDNSDomain());
- fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
+ pw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
}
// Print launch command
if (taskType.equals(TaskType.WORKER) || taskType.equals(
TaskType.PRIMARY_WORKER)) {
- fw.append(parameters.getWorkerLaunchCmd() + '\n');
+ pw.append(parameters.getWorkerLaunchCmd() + '\n');
if (SubmarineLogs.isVerbose()) {
LOG.info(
"Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
}
} else if (taskType.equals(TaskType.PS)) {
- fw.append(parameters.getPSLaunchCmd() + '\n');
+ pw.append(parameters.getPSLaunchCmd() + '\n');
if (SubmarineLogs.isVerbose()) {
LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
@@ -247,7 +254,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
}
}
} finally {
- fw.close();
+ pw.close();
}
return file.getAbsolutePath();
}
@@ -421,18 +428,51 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
return new Artifact().type(Artifact.TypeEnum.DOCKER).id(dockerImageName);
}
+ private void handleQuicklinks(RunJobParameters runJobParameters)
+ throws IOException {
+ List<Quicklink> quicklinks = runJobParameters.getQuicklinks();
+ if (null != quicklinks && !quicklinks.isEmpty()) {
+ for (Quicklink ql : quicklinks) {
+ // Make sure it is a valid instance name
+ String instanceName = ql.getComponentInstanceName();
+ boolean found = false;
+
+ for (Component comp : serviceSpec.getComponents()) {
+ for (int i = 0; i < comp.getNumberOfContainers(); i++) {
+ String possibleInstanceName = comp.getName() + "-" + i;
+ if (possibleInstanceName.equals(instanceName)) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found) {
+ throw new IOException(
+ "Couldn't find a component instance = " + instanceName
+ + " while adding quicklink");
+ }
+
+ String link = ql.getProtocol() + YarnServiceUtils.getDNSName(
+ serviceSpec.getName(), instanceName, getUserName(), getDNSDomain(),
+ ql.getPort());
+ YarnServiceUtils.addQuicklink(serviceSpec, ql.getLabel(), link);
+ }
+ }
+ }
+
private Service createServiceByParameters(RunJobParameters parameters)
throws IOException {
componentToLocalLaunchScriptPath.clear();
- Service service = new Service();
- service.setName(parameters.getName());
- service.setVersion(String.valueOf(System.currentTimeMillis()));
- service.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
+ serviceSpec = new Service();
+ serviceSpec.setName(parameters.getName());
+ serviceSpec.setVersion(String.valueOf(System.currentTimeMillis()));
+ serviceSpec.setArtifact(getDockerArtifact(parameters.getDockerImageName()));
- handleServiceEnvs(service, parameters);
+ handleServiceEnvs(serviceSpec, parameters);
if (parameters.getNumWorkers() > 0) {
- addWorkerComponents(service, parameters);
+ addWorkerComponents(serviceSpec, parameters);
}
if (parameters.getNumPS() > 0) {
@@ -450,7 +490,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
getDockerArtifact(parameters.getPsDockerImage()));
}
handleLaunchCommand(parameters, TaskType.PS, psComponent);
- service.addComponent(psComponent);
+ serviceSpec.addComponent(psComponent);
}
if (parameters.isTensorboardEnabled()) {
@@ -470,14 +510,20 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
// Add tensorboard to quicklink
String tensorboardLink = "http://" + YarnServiceUtils.getDNSName(
- parameters.getName(), TaskType.TENSORBOARD.getComponentName(), 0,
- getUserName(), getDNSDomain(), 6006);
+ parameters.getName(),
+ TaskType.TENSORBOARD.getComponentName() + "-" + 0, getUserName(),
+ getDNSDomain(), 6006);
LOG.info("Link to tensorboard:" + tensorboardLink);
- service.addComponent(tbComponent);
- service.setQuicklinks(ImmutableMap.of("Tensorboard", tensorboardLink));
+ serviceSpec.addComponent(tbComponent);
+
+ YarnServiceUtils.addQuicklink(serviceSpec, TENSORBOARD_QUICKLINK_LABEL,
+ tensorboardLink);
}
- return service;
+ // After all components added, handle quicklinks
+ handleQuicklinks(parameters);
+
+ return serviceSpec;
}
/**
@@ -486,12 +532,11 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
@Override
public ApplicationId submitJob(RunJobParameters parameters)
throws IOException, YarnException {
- Service service = createServiceByParameters(parameters);
+ createServiceByParameters(parameters);
ServiceClient serviceClient = YarnServiceUtils.createServiceClient(
clientContext.getYarnConfig());
- ApplicationId appid = serviceClient.actionCreate(service);
+ ApplicationId appid = serviceClient.actionCreate(serviceSpec);
serviceClient.stop();
- this.serviceSpec = service;
return appid;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
index 9238a67..26402da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
@@ -16,10 +16,20 @@ package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.submarine.common.Envs;
+import org.apache.hadoop.yarn.submarine.common.conf.SubmarineLogs;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
public class YarnServiceUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(YarnServiceUtils.class);
+
// This will be true only in UT.
private static ServiceClient stubServiceClient = null;
@@ -40,10 +50,10 @@ public class YarnServiceUtils {
YarnServiceUtils.stubServiceClient = stubServiceClient;
}
- public static String getDNSName(String serviceName, String componentName,
- int index, String userName, String domain, int port) {
- return componentName + "-" + index + getDNSNameCommonSuffix(serviceName,
- userName, domain, port);
+ public static String getDNSName(String serviceName,
+ String componentInstanceName, String userName, String domain, int port) {
+ return componentInstanceName + getDNSNameCommonSuffix(serviceName, userName,
+ domain, port);
}
private static String getDNSNameCommonSuffix(String serviceName,
@@ -66,12 +76,18 @@ public class YarnServiceUtils {
commonEndpointSuffix) + ",";
String ps = getComponentArrayJson("ps", nPs, commonEndpointSuffix) + "},";
- String task =
- "\\\"task\\\":{" + " \\\"type\\\":\\\"" + curCommponentName + "\\\","
- + " \\\"index\\\":" + '$' + Envs.TASK_INDEX_ENV + "},";
+ StringBuilder sb = new StringBuilder();
+ sb.append("\\\"task\\\":{");
+ sb.append(" \\\"type\\\":\\\"");
+ sb.append(curCommponentName);
+ sb.append("\\\",");
+ sb.append(" \\\"index\\\":");
+ sb.append('$');
+ sb.append(Envs.TASK_INDEX_ENV + "},");
+ String task = sb.toString();
String environment = "\\\"environment\\\":\\\"cloud\\\"}";
- StringBuilder sb = new StringBuilder();
+ sb = new StringBuilder();
sb.append(json);
sb.append(master);
sb.append(worker);
@@ -81,6 +97,21 @@ public class YarnServiceUtils {
return sb.toString();
}
+ public static void addQuicklink(Service serviceSpec, String label,
+ String link) {
+ Map<String, String> quicklinks = serviceSpec.getQuicklinks();
+ if (null == quicklinks) {
+ quicklinks = new HashMap<>();
+ serviceSpec.setQuicklinks(quicklinks);
+ }
+
+ if (SubmarineLogs.isVerbose()) {
+ LOG.info("Added quicklink, " + label + "=" + link);
+ }
+
+ quicklinks.put(label, link);
+ }
+
private static String getComponentArrayJson(String componentName, int count,
String endpointSuffix) {
String component = "\\\"" + componentName + "\\\":";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd63461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
index a88d673..89d39a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.submarine.client.cli.yarnservice;
+import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -100,6 +101,32 @@ public class TestYarnServiceRunJobCli {
Assert.assertTrue(SubmarineLogs.isVerbose());
}
+ private void verifyQuicklink(Service serviceSpec,
+ Map<String, String> expectedQuicklinks) {
+ Map<String, String> actualQuicklinks = serviceSpec.getQuicklinks();
+ if (actualQuicklinks == null || actualQuicklinks.isEmpty()) {
+ Assert.assertTrue(
+ expectedQuicklinks == null || expectedQuicklinks.isEmpty());
+ return;
+ }
+
+ Assert.assertEquals(expectedQuicklinks.size(), actualQuicklinks.size());
+ for (Map.Entry<String, String> expectedEntry : expectedQuicklinks
+ .entrySet()) {
+ Assert.assertTrue(actualQuicklinks.containsKey(expectedEntry.getKey()));
+
+ // $USER could be changed in different environment. so replace $USER by
+ // "user"
+ String expectedValue = expectedEntry.getValue();
+ String actualValue = actualQuicklinks.get(expectedEntry.getKey());
+
+ String userName = System.getProperty("user.name");
+ actualValue = actualValue.replaceAll(userName, "username");
+
+ Assert.assertEquals(expectedValue, actualValue);
+ }
+ }
+
@Test
public void testBasicRunJobForDistributedTraining() throws Exception {
MockClientContext mockClientContext =
@@ -120,6 +147,8 @@ public class TestYarnServiceRunJobCli {
Assert.assertEquals(3, serviceSpec.getComponents().size());
commonVerifyDistributedTrainingSpec(serviceSpec);
+
+ verifyQuicklink(serviceSpec, null);
}
@Test
@@ -147,6 +176,10 @@ public class TestYarnServiceRunJobCli {
verifyTensorboardComponent(runJobCli, serviceSpec,
Resources.createResource(4096, 1));
+
+ verifyQuicklink(serviceSpec, ImmutableMap
+ .of(YarnServiceJobSubmitter.TENSORBOARD_QUICKLINK_LABEL,
+ "http://tensorboard-0.my-job.username.null:6006"));
}
@Test
@@ -232,6 +265,9 @@ public class TestYarnServiceRunJobCli {
verifyTensorboardComponent(runJobCli, serviceSpec,
Resources.createResource(2048, 2));
+ verifyQuicklink(serviceSpec, ImmutableMap
+ .of(YarnServiceJobSubmitter.TENSORBOARD_QUICKLINK_LABEL,
+ "http://tensorboard-0.my-job.username.null:6006"));
}
private void commonTestSingleNodeTraining(Service serviceSpec)
@@ -372,4 +408,62 @@ public class TestYarnServiceRunJobCli {
Assert.assertEquals(jobInfo.get(StorageKeyConstants.INPUT_PATH),
"s3://input");
}
+
+ @Test
+ public void testAddQuicklinksWithoutTensorboard() throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+ "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+ "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+ "ps.image", "--worker_docker_image", "worker.image",
+ "--ps_launch_cmd", "python run-ps.py", "--verbose", "--quicklink",
+ "AAA=http://master-0:8321", "--quicklink",
+ "BBB=http://worker-0:1234" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(3, serviceSpec.getComponents().size());
+
+ commonVerifyDistributedTrainingSpec(serviceSpec);
+
+ verifyQuicklink(serviceSpec, ImmutableMap
+ .of("AAA", "http://master-0.my-job.username.null:8321", "BBB",
+ "http://worker-0.my-job.username.null:1234"));
+ }
+
+ @Test
+ public void testAddQuicklinksWithTensorboard() throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+ "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+ "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+ "ps.image", "--worker_docker_image", "worker.image",
+ "--ps_launch_cmd", "python run-ps.py", "--verbose", "--quicklink",
+ "AAA=http://master-0:8321", "--quicklink",
+ "BBB=http://worker-0:1234", "--tensorboard" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(4, serviceSpec.getComponents().size());
+
+ commonVerifyDistributedTrainingSpec(serviceSpec);
+
+ verifyQuicklink(serviceSpec, ImmutableMap
+ .of("AAA", "http://master-0.my-job.username.null:8321", "BBB",
+ "http://worker-0.my-job.username.null:1234",
+ YarnServiceJobSubmitter.TENSORBOARD_QUICKLINK_LABEL,
+ "http://tensorboard-0.my-job.username.null:6006"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDDS-513. Check if the EventQueue
is not closed before executing handlers. Contributed by Nanda Kumar.
Posted by ha...@apache.org.
HDDS-513. Check if the EventQueue is not closed before executing handlers. Contributed by Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6bb1ca3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6bb1ca3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6bb1ca3
Branch: refs/heads/YARN-1011
Commit: f6bb1ca3c16eb5825188b51d45f32be111266d88
Parents: 2eb597b
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Sep 19 14:35:29 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Sep 19 14:35:29 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/server/events/EventQueue.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6bb1ca3/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
index b2b0df2..9aeab7b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
@@ -55,6 +55,8 @@ public class EventQueue implements EventPublisher, AutoCloseable {
private final AtomicLong eventCount = new AtomicLong(0);
+ private boolean isRunning = true;
+
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void addHandler(
EVENT_TYPE event, EventHandler<PAYLOAD> handler) {
this.addHandler(event, handler, generateHandlerName(handler));
@@ -116,6 +118,10 @@ public class EventQueue implements EventPublisher, AutoCloseable {
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void addHandler(
EVENT_TYPE event, EventExecutor<PAYLOAD> executor,
EventHandler<PAYLOAD> handler) {
+ if (!isRunning) {
+ LOG.warn("Not adding handler for {}, EventQueue is not running", event);
+ return;
+ }
validateEvent(event);
executors.putIfAbsent(event, new HashMap<>());
executors.get(event).putIfAbsent(executor, new ArrayList<>());
@@ -136,6 +142,11 @@ public class EventQueue implements EventPublisher, AutoCloseable {
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
EVENT_TYPE event, PAYLOAD payload) {
+ if (!isRunning) {
+ LOG.warn("Processing of {} is skipped, EventQueue is not running", event);
+ return;
+ }
+
Map<EventExecutor, List<EventHandler>> eventExecutorListMap =
this.executors.get(event);
@@ -187,6 +198,11 @@ public class EventQueue implements EventPublisher, AutoCloseable {
long currentTime = Time.now();
while (true) {
+ if (!isRunning) {
+ LOG.warn("Processing of event skipped. EventQueue is not running");
+ return;
+ }
+
long processed = 0;
Stream<EventExecutor> allExecutor = this.executors.values().stream()
@@ -216,6 +232,8 @@ public class EventQueue implements EventPublisher, AutoCloseable {
public void close() {
+ isRunning = false;
+
Set<EventExecutor> allExecutors = this.executors.values().stream()
.flatMap(handlerMap -> handlerMap.keySet().stream())
.collect(Collectors.toSet());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-15726. Create utility to
limit frequency of log statements. Contributed by Erik Krogen.
Posted by ha...@apache.org.
HADOOP-15726. Create utility to limit frequency of log statements. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a30b4f9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a30b4f9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a30b4f9e
Branch: refs/heads/YARN-1011
Commit: a30b4f9e71cf53f79c38878d6cbe5bbe79bcb277
Parents: 98c9bc4
Author: Chen Liang <cl...@apache.org>
Authored: Wed Sep 19 13:22:37 2018 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Wed Sep 19 13:22:37 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/log/LogThrottlingHelper.java | 358 +++++++++++++++++++
.../hadoop/log/TestLogThrottlingHelper.java | 172 +++++++++
.../hdfs/server/namenode/FSNamesystemLock.java | 46 +--
3 files changed, 547 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
new file mode 100644
index 0000000..aa4e61c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -0,0 +1,358 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.log;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.hadoop.util.Timer;
+
+/**
+ * This is a class to help easily throttle log statements, so that they will
+ * not be emitted more frequently than a certain rate. It is useful to help
+ * prevent flooding the application logs with redundant messages.
+ *
+ * The instantiator specifies a minimum period at which statements should be
+ * logged. When {@link #record(double...)} is called, if enough time has elapsed
+ * since the last time it was called, the return value will indicate to the
+ * caller that it should write to its actual log. Note that this class does not
+ * write to any actual log; it only records information about how many times
+ * {@code record} has been called and with what arguments, and indicates to the
+ * caller whether or not it should write to its log. If not enough time has yet
+ * elapsed, this class records the arguments and updates its summary
+ * information, and indicates to the caller that it should not log.
+ *
+ * For example, say that you want to know whenever too large of a request is
+ * received, but want to avoid flooding the logs if many such requests are
+ * received.
+ * <pre>{@code
+ * // Helper with a minimum period of 5 seconds
+ * private LogThrottlingHelper helper = new LogThrottlingHelper(5000);
+ *
+ * public void receiveRequest(int requestedObjects) {
+ * if (requestedObjects > MAXIMUM_REQUEST_SIZE) {
+ * LogAction logAction = helper.record(requestedObjects);
+ * if (logAction.shouldLog()) {
+ * LOG.warn("Received {} large request(s) with a total of {} objects " +
+ * "requested; maximum objects requested was {}",
+ * logAction.getCount(), logAction.getStats(0).getSum(),
+ * logAction.getStats(0).getMax());
+ * }
+ * }
+ * }
+ * }</pre>
+ * The above snippet allows you to record extraneous events, but if they become
+ * frequent, to limit their presence in the log to only every 5 seconds while
+ * still maintaining overall information about how many large requests were
+ * received.
+ *
+ * <p/>This class can also be used to coordinate multiple logging points; see
+ * {@link #record(String, long, double...)} for more details.
+ *
+ * <p/>This class is not thread-safe.
+ */
+public class LogThrottlingHelper {
+
+ /**
+ * An indication of what action the caller should take. If
+ * {@link #shouldLog()} is false, no other action should be taken, and it is
+ * an error to try to access any of the summary information. If
+ * {@link #shouldLog()} is true, then the caller should write to its log, and
+ * can use the {@link #getCount()} and {@link #getStats(int)} methods to
+ * determine summary information about what has been recorded into this
+ * helper.
+ *
+ * All summary information in this action only represents
+ * {@link #record(double...)} statements which were called <i>after</i> the
+ * last time the caller logged something; that is, since the last time a log
+ * action was returned with a true value for {@link #shouldLog()}. Information
+ * about the {@link #record(double...)} statement which created this log
+ * action is included.
+ */
+ public interface LogAction {
+
+ /**
+ * Return the number of records encapsulated in this action; that is, the
+ * number of times {@code record} was called to produce this action,
+ * including the current one.
+ */
+ int getCount();
+
+ /**
+ * Return summary information for the value that was recorded at index
+ * {@code idx}. Corresponds to the ordering of values passed to
+ * {@link #record(double...)}.
+ */
+ SummaryStatistics getStats(int idx);
+
+ /**
+ * If this is true, the caller should write to its log. Otherwise, the
+ * caller should take no action, and it is an error to call other methods
+ * on this object.
+ */
+ boolean shouldLog();
+
+ }
+
+ /**
+ * A {@link LogAction} representing a state that should not yet be logged.
+ * If any attempt is made to extract information from this, it will throw
+ * an {@link IllegalStateException}.
+ */
+ public static final LogAction DO_NOT_LOG = new NoLogAction();
+ private static final String DEFAULT_RECORDER_NAME =
+ "__DEFAULT_RECORDER_NAME__";
+
+ /**
+ * This throttler will not trigger log statements more frequently than this
+ * period.
+ */
+ private final long minLogPeriodMs;
+ /**
+ * The name of the recorder treated as the primary; this is the only one which
+ * will trigger logging. Other recorders are dependent on the state of this
+ * recorder. This may be null, in which case a primary has not yet been set.
+ */
+ private String primaryRecorderName;
+ private final Timer timer;
+ private final Map<String, LoggingAction> currentLogs;
+
+ private long lastLogTimestampMs = Long.MIN_VALUE;
+
+ /**
+ * Create a log helper without any primary recorder.
+ *
+ * @see #LogThrottlingHelper(long, String)
+ */
+ public LogThrottlingHelper(long minLogPeriodMs) {
+ this(minLogPeriodMs, null);
+ }
+
+ /**
+ * Create a log helper with a specified primary recorder name; this can be
+ * used in conjunction with {@link #record(String, long, double...)} to set up
+ * primary and dependent recorders. See
+ * {@link #record(String, long, double...)} for more details.
+ *
+ * @param minLogPeriodMs The minimum period with which to log; do not log
+ * more frequently than this.
+ * @param primaryRecorderName The name of the primary recorder.
+ */
+ public LogThrottlingHelper(long minLogPeriodMs, String primaryRecorderName) {
+ this(minLogPeriodMs, primaryRecorderName, new Timer());
+ }
+
+ @VisibleForTesting
+ LogThrottlingHelper(long minLogPeriodMs, String primaryRecorderName,
+ Timer timer) {
+ this.minLogPeriodMs = minLogPeriodMs;
+ this.primaryRecorderName = primaryRecorderName;
+ this.timer = timer;
+ this.currentLogs = new HashMap<>();
+ }
+
+ /**
+ * Record some set of values at the current time into this helper. Note that
+ * this does <i>not</i> actually write information to any log. Instead, this
+ * will return a LogAction indicating whether or not the caller should write
+ * to its own log. The LogAction will additionally contain summary information
+ * about the values specified since the last time the caller was expected to
+ * write to its log.
+ *
+ * <p/>Specifying multiple values will maintain separate summary statistics
+ * about each value. For example:
+ * <pre>{@code
+ * helper.record(1, 0);
+ * LogAction action = helper.record(3, 100);
+ * action.getStats(0); // == 2
+ * action.getStats(1); // == 50
+ * }</pre>
+ *
+ * @param values The values about which to maintain summary information. Every
+ * time this method is called, the same number of values must
+ * be specified.
+ * @return A LogAction indicating whether or not the caller should write to
+ * its log.
+ */
+ public LogAction record(double... values) {
+ return record(DEFAULT_RECORDER_NAME, timer.monotonicNow(), values);
+ }
+
+ /**
+ * Record some set of values at the specified time into this helper. This can
+ * be useful to avoid fetching the current time twice if the caller has
+ * already done so for other purposes. This additionally allows the caller to
+ * specify a name for this recorder. When multiple names are used, one is
+ * denoted as the primary recorder. Only recorders named as the primary
+ * will trigger logging; other names not matching the primary can <i>only</i>
+ * be triggered by following the primary. This is used to coordinate multiple
+ * logging points. A primary can be set via the
+ * {@link #LogThrottlingHelper(long, String)} constructor. If no primary
+ * is set in the constructor, then the first recorder name used becomes the
+ * primary.
+ *
+ * If multiple names are used, they maintain entirely different sets of values
+ * and summary information. For example:
+ * <pre>{@code
+ * // Initialize "pre" as the primary recorder name
+ * LogThrottlingHelper helper = new LogThrottlingHelper(1000, "pre");
+ * LogAction preLog = helper.record("pre", Time.monotonicNow());
+ * if (preLog.shouldLog()) {
+ * // ...
+ * }
+ * double eventsProcessed = ... // perform some action
+ * LogAction postLog =
+ * helper.record("post", Time.monotonicNow(), eventsProcessed);
+ * if (postLog.shouldLog()) {
+ * // ...
+ * // Can use postLog.getStats(0) to access eventsProcessed information
+ * }
+ * }</pre>
+ * Since "pre" is the primary recorder name, logging to "pre" will trigger a
+ * log action if enough time has elapsed. This will indicate that "post"
+ * should log as well. This ensures that "post" is always logged in the same
+ * iteration as "pre", yet each one is able to maintain its own summary
+ * information.
+ *
+ * <p/>Other behavior is the same as {@link #record(double...)}.
+ *
+ * @param recorderName The name of the recorder. This is used to check if the
+ * current recorder is the primary. Other names are
+ * arbitrary and are only used to differentiate between
+ * distinct recorders.
+ * @param currentTimeMs The current time.
+ * @param values The values to log.
+ * @return The LogAction for the specified recorder.
+ *
+ * @see #record(double...)
+ */
+ public LogAction record(String recorderName, long currentTimeMs,
+ double... values) {
+ if (primaryRecorderName == null) {
+ primaryRecorderName = recorderName;
+ }
+ LoggingAction currentLog = currentLogs.get(recorderName);
+ if (currentLog == null || currentLog.hasLogged()) {
+ currentLog = new LoggingAction(values.length);
+ if (!currentLogs.containsKey(recorderName)) {
+ // Always log newly created loggers
+ currentLog.setShouldLog();
+ }
+ currentLogs.put(recorderName, currentLog);
+ }
+ currentLog.recordValues(values);
+ if (primaryRecorderName.equals(recorderName) &&
+ currentTimeMs - minLogPeriodMs >= lastLogTimestampMs) {
+ lastLogTimestampMs = currentTimeMs;
+ for (LoggingAction log : currentLogs.values()) {
+ log.setShouldLog();
+ }
+ }
+ if (currentLog.shouldLog()) {
+ currentLog.setHasLogged();
+ return currentLog;
+ } else {
+ return DO_NOT_LOG;
+ }
+ }
+
+ /**
+ * A standard log action which keeps track of all of the values which have
+ * been logged. This is also used for internal bookkeeping via its private
+ * fields and methods; it will maintain whether or not it is ready to be
+ * logged ({@link #shouldLog()}) as well as whether or not it has been
+ * returned for logging yet ({@link #hasLogged()}).
+ */
+ private static class LoggingAction implements LogAction {
+
+ private int count = 0;
+ private final SummaryStatistics[] stats;
+ private boolean shouldLog = false;
+ private boolean hasLogged = false;
+
+ LoggingAction(int valueCount) {
+ stats = new SummaryStatistics[valueCount];
+ for (int i = 0; i < stats.length; i++) {
+ stats[i] = new SummaryStatistics();
+ }
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public SummaryStatistics getStats(int idx) {
+ if (idx < 0 || idx >= stats.length) {
+ throw new IllegalArgumentException("Requested stats at idx " + idx +
+ " but this log only maintains " + stats.length + " stats");
+ }
+ return stats[idx];
+ }
+
+ public boolean shouldLog() {
+ return shouldLog;
+ }
+
+ private void setShouldLog() {
+ shouldLog = true;
+ }
+
+ private boolean hasLogged() {
+ return hasLogged;
+ }
+
+ private void setHasLogged() {
+ hasLogged = true;
+ }
+
+ private void recordValues(double... values) {
+ if (values.length != stats.length) {
+ throw new IllegalArgumentException("received " + values.length +
+ " values but expected " + stats.length);
+ }
+ count++;
+ for (int i = 0; i < values.length; i++) {
+ stats[i].addValue(values[i]);
+ }
+ }
+
+ }
+
+ /**
+ * A non-logging action.
+ *
+ * @see #DO_NOT_LOG
+ */
+ private static class NoLogAction implements LogAction {
+
+ public int getCount() {
+ throw new IllegalStateException("Cannot be logged yet!");
+ }
+
+ public SummaryStatistics getStats(int idx) {
+ throw new IllegalStateException("Cannot be logged yet!");
+ }
+
+ public boolean shouldLog() {
+ return false;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
new file mode 100644
index 0000000..a675d0a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.log;
+
+import org.apache.hadoop.log.LogThrottlingHelper.LogAction;
+import org.apache.hadoop.util.FakeTimer;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link LogThrottlingHelper}.
+ */
+public class TestLogThrottlingHelper {
+
+ private static final int LOG_PERIOD = 100;
+
+ private LogThrottlingHelper helper;
+ private FakeTimer timer;
+
+ @Before
+ public void setup() {
+ timer = new FakeTimer();
+ helper = new LogThrottlingHelper(LOG_PERIOD, null, timer);
+ }
+
+ @Test
+ public void testBasicLogging() {
+ assertTrue(helper.record().shouldLog());
+
+ for (int i = 0; i < 5; i++) {
+ timer.advance(LOG_PERIOD / 10);
+ assertFalse(helper.record().shouldLog());
+ }
+ timer.advance(LOG_PERIOD);
+ assertTrue(helper.record().shouldLog());
+ }
+
+ @Test
+ public void testLoggingWithValue() {
+ assertTrue(helper.record(1).shouldLog());
+
+ for (int i = 0; i < 4; i++) {
+ timer.advance(LOG_PERIOD / 5);
+ assertFalse(helper.record(i % 2 == 0 ? 0 : 1).shouldLog());
+ }
+
+ timer.advance(LOG_PERIOD);
+ LogAction action = helper.record(0.5);
+ assertTrue(action.shouldLog());
+ assertEquals(5, action.getCount());
+ assertEquals(0.5, action.getStats(0).getMean(), 0.01);
+ assertEquals(1.0, action.getStats(0).getMax(), 0.01);
+ assertEquals(0.0, action.getStats(0).getMin(), 0.01);
+ }
+
+ @Test
+ public void testLoggingWithMultipleValues() {
+ assertTrue(helper.record(1).shouldLog());
+
+ for (int i = 0; i < 4; i++) {
+ timer.advance(LOG_PERIOD / 5);
+ int base = i % 2 == 0 ? 0 : 1;
+ assertFalse(helper.record(base, base * 2).shouldLog());
+ }
+
+ timer.advance(LOG_PERIOD);
+ LogAction action = helper.record(0.5, 1.0);
+ assertTrue(action.shouldLog());
+ assertEquals(5, action.getCount());
+ for (int i = 1; i <= 2; i++) {
+ assertEquals(0.5 * i, action.getStats(i - 1).getMean(), 0.01);
+ assertEquals(1.0 * i, action.getStats(i - 1).getMax(), 0.01);
+ assertEquals(0.0, action.getStats(i - 1).getMin(), 0.01);
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testLoggingWithInconsistentValues() {
+ assertTrue(helper.record(1, 2).shouldLog());
+ helper.record(1, 2);
+ helper.record(1, 2, 3);
+ }
+
+ @Test
+ public void testNamedLoggersWithoutSpecifiedPrimary() {
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+
+ assertFalse(helper.record("foo", LOG_PERIOD / 2).shouldLog());
+ assertFalse(helper.record("bar", LOG_PERIOD / 2).shouldLog());
+
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+
+ assertFalse(helper.record("foo", (LOG_PERIOD * 3) / 2).shouldLog());
+ assertFalse(helper.record("bar", (LOG_PERIOD * 3) / 2).shouldLog());
+
+ assertFalse(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("foo", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ }
+
+ @Test
+ public void testPrimaryAndDependentLoggers() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+ assertFalse(helper.record("bar", 0).shouldLog());
+ assertFalse(helper.record("foo", 0).shouldLog());
+
+ assertFalse(helper.record("foo", LOG_PERIOD / 2).shouldLog());
+ assertFalse(helper.record("bar", LOG_PERIOD / 2).shouldLog());
+
+ // Both should log once the period has elapsed
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+
+ // "bar" should not log yet because "foo" hasn't been triggered
+ assertFalse(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("foo", LOG_PERIOD * 2).shouldLog());
+ // The timing of "bar" shouldn't matter as it is dependent on "foo"
+ assertTrue(helper.record("bar", 0).shouldLog());
+ }
+
+ @Test
+ public void testMultipleLoggersWithValues() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0, 2).shouldLog());
+ assertTrue(helper.record("baz", 0, 3, 3).shouldLog());
+
+ // "bar"/"baz" should not log yet because "foo" hasn't been triggered
+ assertFalse(helper.record("bar", LOG_PERIOD, 2).shouldLog());
+ assertFalse(helper.record("baz", LOG_PERIOD, 3, 3).shouldLog());
+
+ // All should log once the period has elapsed
+ LogAction foo = helper.record("foo", LOG_PERIOD);
+ LogAction bar = helper.record("bar", LOG_PERIOD, 2);
+ LogAction baz = helper.record("baz", LOG_PERIOD, 3, 3);
+ assertTrue(foo.shouldLog());
+ assertTrue(bar.shouldLog());
+ assertTrue(baz.shouldLog());
+ assertEquals(1, foo.getCount());
+ assertEquals(2, bar.getCount());
+ assertEquals(2, baz.getCount());
+ assertEquals(2.0, bar.getStats(0).getMean(), 0.01);
+ assertEquals(3.0, baz.getStats(0).getMean(), 0.01);
+ assertEquals(3.0, baz.getStats(1).getMean(), 0.01);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index 5992e54..7c28465 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.log.LogThrottlingHelper;
import org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Timer;
@@ -40,6 +41,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORT
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.log.LogThrottlingHelper.LogAction;
/**
* Mimics a ReentrantReadWriteLock but does not directly implement the interface
@@ -74,11 +76,8 @@ class FSNamesystemLock {
private final long writeLockReportingThresholdMs;
/** Last time stamp for write lock. Keep the longest one for multi-entrance.*/
private long writeLockHeldTimeStampNanos;
- private int numWriteLockWarningsSuppressed = 0;
- /** Time stamp (ms) of the last time a write lock report was written. */
- private long timeStampOfLastWriteLockReportMs = 0;
- /** Longest time (ms) a write lock was held since the last report. */
- private long longestWriteLockHeldIntervalMs = 0;
+ /** Frequency limiter used for reporting long write lock hold times. */
+ private final LogThrottlingHelper writeLockReportLogger;
/** Threshold (ms) for long holding read lock report. */
private final long readLockReportingThresholdMs;
@@ -132,6 +131,8 @@ class FSNamesystemLock {
this.lockSuppressWarningIntervalMs = conf.getTimeDuration(
DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY,
DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
+ this.writeLockReportLogger =
+ new LogThrottlingHelper(lockSuppressWarningIntervalMs);
this.metricsEnabled = conf.getBoolean(
DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY,
DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT);
@@ -251,25 +252,11 @@ class FSNamesystemLock {
final long writeLockIntervalMs =
TimeUnit.NANOSECONDS.toMillis(writeLockIntervalNanos);
- boolean logReport = false;
- int numSuppressedWarnings = 0;
- long longestLockHeldIntervalMs = 0;
+ LogAction logAction = LogThrottlingHelper.DO_NOT_LOG;
if (needReport &&
writeLockIntervalMs >= this.writeLockReportingThresholdMs) {
- if (writeLockIntervalMs > longestWriteLockHeldIntervalMs) {
- longestWriteLockHeldIntervalMs = writeLockIntervalMs;
- }
- if (currentTimeMs - timeStampOfLastWriteLockReportMs >
- this.lockSuppressWarningIntervalMs) {
- logReport = true;
- numSuppressedWarnings = numWriteLockWarningsSuppressed;
- numWriteLockWarningsSuppressed = 0;
- longestLockHeldIntervalMs = longestWriteLockHeldIntervalMs;
- longestWriteLockHeldIntervalMs = 0;
- timeStampOfLastWriteLockReportMs = currentTimeMs;
- } else {
- numWriteLockWarningsSuppressed++;
- }
+ logAction = writeLockReportLogger
+ .record("write", currentTimeMs, writeLockIntervalMs);
}
coarseLock.writeLock().unlock();
@@ -278,13 +265,14 @@ class FSNamesystemLock {
addMetric(opName, writeLockIntervalNanos, true);
}
- if (logReport) {
- FSNamesystem.LOG.info("FSNamesystem write lock held for " +
- writeLockIntervalMs + " ms via\n" +
- StringUtils.getStackTrace(Thread.currentThread()) +
- "\tNumber of suppressed write-lock reports: " +
- numSuppressedWarnings + "\n\tLongest write-lock held interval: " +
- longestLockHeldIntervalMs);
+ if (logAction.shouldLog()) {
+ FSNamesystem.LOG.info("FSNamesystem write lock held for {} ms via {}\t" +
+ "Number of suppressed write-lock reports: {}\n\tLongest write-lock " +
+ "held interval: {} \n\tTotal suppressed write-lock held time: {}",
+ writeLockIntervalMs,
+ StringUtils.getStackTrace(Thread.currentThread()),
+ logAction.getCount() - 1, logAction.getStats(0).getMax(),
+ logAction.getStats(0).getSum() - writeLockIntervalMs);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-6675. Add NM support to
launch opportunistic containers based on overallocation. Contributed by Haibo
Chen.
Posted by ha...@apache.org.
YARN-6675. Add NM support to launch opportunistic containers based on overallocation. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08116cc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08116cc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08116cc4
Branch: refs/heads/YARN-1011
Commit: 08116cc419126cfecb886aab62642de817a4b885
Parents: 0067800
Author: Miklos Szegedi <sz...@apache.org>
Authored: Fri Apr 20 14:15:29 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:19:25 2018 -0700
----------------------------------------------------------------------
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 8 +-
.../launcher/ContainerLaunch.java | 2 +-
.../launcher/ContainersLauncher.java | 9 +-
.../monitor/ContainersMonitor.java | 38 +-
.../monitor/ContainersMonitorImpl.java | 56 +-
.../AllocationBasedResourceTracker.java | 114 ++
...locationBasedResourceUtilizationTracker.java | 158 ---
.../scheduler/ContainerScheduler.java | 317 +++--
.../scheduler/ContainerSchedulerEventType.java | 4 +-
.../scheduler/NMAllocationPolicy.java | 63 +
.../scheduler/ResourceUtilizationTracker.java | 17 +-
.../SnapshotBasedOverAllocationPolicy.java | 54 +
.../UtilizationBasedResourceTracker.java | 95 ++
.../BaseContainerManagerTest.java | 35 +
.../TestContainersMonitorResourceChange.java | 9 +-
.../TestAllocationBasedResourceTracker.java | 82 ++
...locationBasedResourceUtilizationTracker.java | 93 --
.../TestContainerSchedulerRecovery.java | 58 +-
...estContainerSchedulerWithOverAllocation.java | 1121 ++++++++++++++++++
20 files changed, 1916 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 572684e..d757376 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -543,7 +543,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private ResourceUtilization getContainersUtilization() {
ContainersMonitor containersMonitor =
this.context.getContainerManager().getContainersMonitor();
- return containersMonitor.getContainersUtilization();
+ return containersMonitor.getContainersUtilization(false).getUtilization();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 27a7c80..a08e227 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -245,6 +245,12 @@ public class ContainerManagerImpl extends CompositeService implements
metrics);
addService(rsrcLocalizationSrvc);
+ this.containersMonitor = createContainersMonitor(exec);
+ addService(this.containersMonitor);
+
+ // ContainersLauncher must be added after ContainersMonitor
+ // because the former depends on the latter to initialize
+ // over-allocation first.
containersLauncher = createContainersLauncher(context, exec);
addService(containersLauncher);
@@ -269,8 +275,6 @@ public class ContainerManagerImpl extends CompositeService implements
nmMetricsPublisher = createNMTimelinePublisher(context);
context.setNMTimelinePublisher(nmMetricsPublisher);
}
- this.containersMonitor = createContainersMonitor(exec);
- addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
new ContainerEventDispatcher());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6347d4e..0228332 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1079,7 +1079,7 @@ public class ContainerLaunch implements Callable<Integer> {
* @return Process ID
* @throws Exception
*/
- private String getContainerPid(Path pidFilePath) throws Exception {
+ protected String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 7870f86..2f5acfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -121,8 +121,7 @@ public class ContainersLauncher extends AbstractService
containerId.getApplicationAttemptId().getApplicationId());
ContainerLaunch launch =
- new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
- event.getContainer(), dirsHandler, containerManager);
+ createContainerLaunch(app, event.getContainer());
containerLauncher.submit(launch);
running.put(containerId, launch);
break;
@@ -225,4 +224,10 @@ public class ContainersLauncher extends AbstractService
break;
}
}
+
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index 64831e9..8da4ec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -23,10 +23,24 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
public interface ContainersMonitor extends Service,
EventHandler<ContainersMonitorEvent>, ResourceView {
- ResourceUtilization getContainersUtilization();
+
+ /**
+ * Get the aggregate resource utilization of containers running on the node,
+ * with a timestamp of the measurement.
+ * @param latest true if the latest result should be returned
+ * @return ResourceUtilization resource utilization of all containers
+ */
+ ContainersResourceUtilization getContainersUtilization(boolean latest);
+
+ /**
+ * Get the policy to over-allocate containers when over-allocation is on.
+ * @return null if over-allocation is turned off
+ */
+ NMAllocationPolicy getContainerOverAllocationPolicy();
float getVmemRatio();
@@ -66,4 +80,26 @@ public interface ContainersMonitor extends Service,
* containersMonitor.getVmemRatio());
resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores);
}
+
+ /**
+ * A snapshot of resource utilization of all containers with the timestamp.
+ */
+ final class ContainersResourceUtilization {
+ private final ResourceUtilization utilization;
+ private final long timestamp;
+
+ public ContainersResourceUtilization(
+ ResourceUtilization utilization, long timestamp) {
+ this.utilization = utilization;
+ this.timestamp = timestamp;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public ResourceUtilization getUtilization() {
+ return utilization;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 7873882..a045d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,8 +116,9 @@ public class ContainersMonitorImpl extends AbstractService implements
CPU, MEMORY
}
- private ResourceUtilization containersUtilization;
+ private ContainersResourceUtilization latestContainersUtilization;
+ private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
@@ -129,7 +134,8 @@ public class ContainersMonitorImpl extends AbstractService implements
this.monitoringThread = new MonitoringThread();
- this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ ResourceUtilization.newInstance(-1, -1, -1.0f), -1L);
}
@Override
@@ -365,6 +371,10 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
cpuPreemptionThreshold, memoryPreemptionThreshold);
+ // TODO make this configurable
+ this.overAllocationPolicy =
+ createOverAllocationPolicy(resourceThresholds);
+
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -372,6 +382,11 @@ public class ContainersMonitorImpl extends AbstractService implements
cpuPreemptionThreshold + ")");
}
+ protected NMAllocationPolicy createOverAllocationPolicy(
+ ResourceThresholds resourceThresholds) {
+ return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -655,7 +670,12 @@ public class ContainersMonitorImpl extends AbstractService implements
}
// Save the aggregated utilization of the containers
- setContainersUtilization(trackedContainersUtilization);
+ setLatestContainersUtilization(trackedContainersUtilization);
+
+ // check opportunity to start containers if over-allocation is on
+ if (context.isOverAllocationEnabled()) {
+ attemptToStartContainersUponLowUtilization();
+ }
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1045,12 +1065,34 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
- public ResourceUtilization getContainersUtilization() {
- return this.containersUtilization;
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ // TODO update containerUtilization if latest is true
+ return this.latestContainersUtilization;
+ }
+
+ @Override
+ public NMAllocationPolicy getContainerOverAllocationPolicy() {
+ return overAllocationPolicy;
+ }
+
+ private void setLatestContainersUtilization(ResourceUtilization utilization) {
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ utilization, System.currentTimeMillis());
}
- private void setContainersUtilization(ResourceUtilization utilization) {
- this.containersUtilization = utilization;
+ @VisibleForTesting
+ public void attemptToStartContainersUponLowUtilization() {
+ if (getContainerOverAllocationPolicy() != null) {
+ Resource available = getContainerOverAllocationPolicy()
+ .getAvailableResources();
+ if (available.getMemorySize() > 0 &&
+ available.getVirtualCores() > 0) {
+ eventDispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(null,
+ ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
new file mode 100644
index 0000000..86b3698
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the resource utilization tracker that equates
+ * resource utilization with the total resource allocated to the container.
+ */
+public class AllocationBasedResourceTracker
+ implements ResourceUtilizationTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private static final Resource UNAVAILABLE =
+ Resource.newInstance(0, 0);
+
+ private ResourceUtilization containersAllocation;
+ private ContainerScheduler scheduler;
+
+
+ AllocationBasedResourceTracker(ContainerScheduler scheduler) {
+ this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * Get the accumulation of totally allocated resources to containers.
+ * @return ResourceUtilization Resource Utilization.
+ */
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return this.containersAllocation;
+ }
+
+ /**
+ * Get the amount of resources that have not been allocated to containers.
+ * @return Resource resources that have not been allocated to containers.
+ */
+ protected Resource getUnallocatedResources() {
+ // unallocated resources = node capacity - containers allocation
+ // = -(container allocation - node capacity)
+ ResourceUtilization allocationClone =
+ ResourceUtilization.newInstance(containersAllocation);
+ getContainersMonitor()
+ .subtractNodeResourcesFromResourceUtilization(allocationClone);
+
+ Resource unallocated = UNAVAILABLE;
+ if (allocationClone.getCPU() <= 0 &&
+ allocationClone.getPhysicalMemory() <= 0 &&
+ allocationClone.getVirtualMemory() <= 0) {
+ int cpu = Math.round(allocationClone.getCPU() *
+ getContainersMonitor().getVCoresAllocatedForContainers());
+ long memory = allocationClone.getPhysicalMemory();
+ unallocated = Resource.newInstance(-memory, -cpu);
+ }
+ return unallocated;
+ }
+
+
+ @Override
+ public Resource getAvailableResources() {
+ return getUnallocatedResources();
+ }
+
+ /**
+ * Add Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerLaunched(Container container) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Subtract Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerReleased(Container container) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ public ContainersMonitor getContainersMonitor() {
+ return this.scheduler.getContainersMonitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 6e2b617..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An implementation of the {@link ResourceUtilizationTracker} that equates
- * resource utilization with the total resource allocated to the container.
- */
-public class AllocationBasedResourceUtilizationTracker implements
- ResourceUtilizationTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
-
- private ResourceUtilization containersAllocation;
- private ContainerScheduler scheduler;
-
- AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) {
- this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
- this.scheduler = scheduler;
- }
-
- /**
- * Get the accumulation of totally allocated resources to a container.
- * @return ResourceUtilization Resource Utilization.
- */
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return this.containersAllocation;
- }
-
- /**
- * Add Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void addContainerResources(Container container) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Subtract Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void subtractContainerResource(Container container) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Check if NM has resources available currently to run the container.
- * @param container Container.
- * @return True, if NM has resources available currently to run the container.
- */
- @Override
- public boolean hasResourcesAvailable(Container container) {
- long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
- return hasResourcesAvailable(pMemBytes,
- (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
- container.getResource().getVirtualCores());
- }
-
- private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
- int cpuVcores) {
- // Check physical memory.
- if (LOG.isDebugEnabled()) {
- LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
- this.containersAllocation.getPhysicalMemory(),
- (pMemBytes >> 20),
- (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
- }
- if (this.containersAllocation.getPhysicalMemory() +
- (int) (pMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getPmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before vMemCheck" +
- "[isEnabled={}, current={} + asked={} > allowed={}]",
- getContainersMonitor().isVmemCheckEnabled(),
- this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
- (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
- }
- // Check virtual memory.
- if (getContainersMonitor().isVmemCheckEnabled() &&
- this.containersAllocation.getVirtualMemory() +
- (int) (vMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getVmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before cpuCheck [asked={} > allowed={}]",
- this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers());
- }
- // Check CPU. Compare using integral values of cores to avoid decimal
- // inaccuracies.
- if (!hasEnoughCpu(this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers(), cpuVcores)) {
- return false;
- }
- return true;
- }
-
- /**
- * Returns whether there is enough space for coresRequested in totalCores.
- * Converts currentAllocation usage to nearest integer count before comparing,
- * as floats are inherently imprecise. NOTE: this calculation assumes that
- * requested core counts must be integers, and currentAllocation core count
- * must also be an integer.
- *
- * @param currentAllocation The current allocation, a float value from 0 to 1.
- * @param totalCores The total cores in the system.
- * @param coresRequested The number of cores requested.
- * @return True if currentAllocationtotalCores*coresRequested <=
- * totalCores.
- */
- public boolean hasEnoughCpu(float currentAllocation, long totalCores,
- int coresRequested) {
- // Must not cast here, as it would truncate the decimal digits.
- return Math.round(currentAllocation * totalCores)
- + coresRequested <= totalCores;
- }
-
- public ContainersMonitor getContainersMonitor() {
- return this.scheduler.getContainersMonitor();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index a61b9d1..0bebe44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +82,10 @@ public class ContainerScheduler extends AbstractService implements
// Queue of Guaranteed Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedGuaranteedContainers = new LinkedHashMap<>();
+ // sum of the resources requested by guaranteed containers in queue
+ private final Resource guaranteedResourcesDemanded =
+ Resource.newInstance(0, 0);
+
// Queue of Opportunistic Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedOpportunisticContainers = new LinkedHashMap<>();
@@ -88,6 +94,10 @@ public class ContainerScheduler extends AbstractService implements
// or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
+ // sum of the resources to be released by opportunistic containers that
+ // have been marked to be killed or paused.
+ private final Resource opportunisticResourcesToBeReleased =
+ Resource.newInstance(0, 0);
// Containers launched by the Scheduler will take a while to actually
// move to the RUNNING state, but should still be fair game for killing
@@ -128,6 +138,17 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
+ @VisibleForTesting
+ public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+ NodeManagerMetrics metrics, int qLength) {
+ super(ContainerScheduler.class.getName());
+ this.context = context;
+ this.dispatcher = dispatcher;
+ this.metrics = metrics;
+ this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
+ }
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -155,20 +176,16 @@ public class ContainerScheduler extends AbstractService implements
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
YarnConfiguration.
DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
+ // We assume over allocation configurations have been initialized
+ this.utilizationTracker = getResourceTracker();
}
- @VisibleForTesting
- public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
- NodeManagerMetrics metrics, int qLength) {
- super(ContainerScheduler.class.getName());
- this.context = context;
- this.dispatcher = dispatcher;
- this.metrics = metrics;
- this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
- this.utilizationTracker =
- new AllocationBasedResourceUtilizationTracker(this);
- this.opportunisticContainersStatus =
- OpportunisticContainersStatus.newInstance();
+ private AllocationBasedResourceTracker getResourceTracker() {
+ if (context.isOverAllocationEnabled()) {
+ return new UtilizationBasedResourceTracker(this);
+ } else {
+ return new AllocationBasedResourceTracker(this);
+ }
}
/**
@@ -191,14 +208,18 @@ public class ContainerScheduler extends AbstractService implements
if (event instanceof UpdateContainerSchedulerEvent) {
onUpdateContainer((UpdateContainerSchedulerEvent) event);
} else {
- LOG.error("Unknown event type on UpdateCOntainer: " + event.getType());
+ LOG.error("Unknown event type on UpdateContainer: " + event.getType());
}
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
case RECOVERY_COMPLETED:
- startPendingContainers(maxOppQueueLength <= 0);
+ startPendingContainers(false);
+ break;
+ case SCHEDULE_CONTAINERS:
+ startPendingContainers(true);
+ break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -213,10 +234,10 @@ public class ContainerScheduler extends AbstractService implements
ContainerId containerId = updateEvent.getContainer().getContainerId();
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
- this.utilizationTracker.subtractContainerResource(
+ this.utilizationTracker.containerReleased(
new ContainerImpl(getConfig(), null, null, null, null,
updateEvent.getOriginalToken(), context));
- this.utilizationTracker.addContainerResources(
+ this.utilizationTracker.containerLaunched(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
@@ -232,17 +253,20 @@ public class ContainerScheduler extends AbstractService implements
if (queuedOpportunisticContainers.remove(containerId) != null) {
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
- //Kill/pause opportunistic containers if any to make room for
- // promotion request
- reclaimOpportunisticContainerResources(updateEvent.getContainer());
+ Resources.addTo(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
+ startPendingContainers(true);
}
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
// containers
if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.subtractFrom(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
queuedOpportunisticContainers.put(containerId,
updateEvent.getContainer());
+ startPendingContainers(false);
}
}
try {
@@ -269,6 +293,7 @@ public class ContainerScheduler extends AbstractService implements
|| rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else if (execType == ExecutionType.OPPORTUNISTIC) {
queuedOpportunisticContainers
.put(container.getContainerId(), container);
@@ -279,7 +304,7 @@ public class ContainerScheduler extends AbstractService implements
}
} else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
- utilizationTracker.addContainerResources(container);
+ utilizationTracker.containerLaunched(container);
}
if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
&& rcs.getCapability() != null) {
@@ -344,65 +369,107 @@ public class ContainerScheduler extends AbstractService implements
}
private void onResourcesReclaimed(Container container) {
- oppContainersToKill.remove(container.getContainerId());
+ ContainerId containerId = container.getContainerId();
+
+ // This could be killed externally for eg. by the ContainerManager,
+ // in which case, the container might still be queued.
+ if (queuedOpportunisticContainers.remove(containerId) != null) {
+ return;
+ }
// This could be killed externally for eg. by the ContainerManager,
// in which case, the container might still be queued.
- Container queued =
- queuedOpportunisticContainers.remove(container.getContainerId());
- if (queued == null) {
- queuedGuaranteedContainers.remove(container.getContainerId());
+ if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
+ return;
+ }
+
+ if (oppContainersToKill.remove(containerId) != null) {
+ Resources.subtractFrom(
+ opportunisticResourcesToBeReleased, container.getResource());
}
// Requeue PAUSED containers
if (container.getContainerState() == ContainerState.PAUSED) {
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
- queuedGuaranteedContainers.put(container.getContainerId(), container);
+ queuedGuaranteedContainers.put(containerId, container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else {
- queuedOpportunisticContainers.put(
- container.getContainerId(), container);
+ queuedOpportunisticContainers.put(containerId, container);
}
}
// decrement only if it was a running container
- Container completedContainer = runningContainers.remove(container
- .getContainerId());
+ Container completedContainer = runningContainers.remove(containerId);
// only a running container releases resources upon completion
boolean resourceReleased = completedContainer != null;
if (resourceReleased) {
- this.utilizationTracker.subtractContainerResource(container);
+ this.utilizationTracker.containerReleased(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
+
+ // In case of over-allocation being turned on, we may need to reclaim
+ // more resources since the opportunistic containers that have been
+ // killed or paused may have not released as much resource as we need.
+ boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
+ startPendingContainers(reclaimOpportunisticResources);
}
}
/**
* Start pending containers in the queue.
- * @param forceStartGuaranteedContaieners When this is true, start guaranteed
- * container without looking at available resource
+ * @param reclaimOpportunisticResources if set to true, resources allocated
+ * to running OPPORTUNISTIC containers will be reclaimed in
+ * cases where there are GUARANTEED containers being queued
*/
- private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
- // Start guaranteed containers that are paused, if resources available.
- boolean resourcesAvailable = startContainers(
- queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
- // Start opportunistic containers, if resources available.
- if (resourcesAvailable) {
- startContainers(queuedOpportunisticContainers.values(), false);
+ private void startPendingContainers(boolean reclaimOpportunisticResources) {
+ // When opportunistic container not allowed (which is determined by
+ // max-queue length of pending opportunistic containers <= 0), start
+ // guaranteed containers without looking at available resources and
+ // skip scanning the queue of opportunistic containers
+ if (maxOppQueueLength <= 0) {
+ forcefullyStartGuaranteedContainers();
+ return;
+ }
+
+ Resource available = utilizationTracker.getAvailableResources();
+
+ // Start guaranteed containers that are queued, if resources available.
+ boolean allGuaranteedContainersLaunched =
+ startGuaranteedContainers(available);
+ // Start opportunistic containers, if resources available, which is true
+ // if all guaranteed containers in queue have been launched.
+ if (allGuaranteedContainersLaunched) {
+ startOpportunisticContainers(available);
+ } else {
+ // If not all guaranteed containers in queue are launched, we may need
+ // to reclaim resources from opportunistic containers that are running.
+ if (reclaimOpportunisticResources) {
+ reclaimOpportunisticContainerResources();
+ }
}
}
- private boolean startContainers(
- Collection<Container> containersToBeStarted, boolean force) {
- Iterator<Container> cIter = containersToBeStarted.iterator();
+ /**
+ * Try to launch as many GUARANTEED containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all queued GUARANTEED containers are launched
+ * or there is no GUARANTEED containers to launch
+ */
+ private boolean startGuaranteedContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (tryStartContainer(container, force)) {
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
} else {
resourcesAvailable = false;
}
@@ -410,25 +477,49 @@ public class ContainerScheduler extends AbstractService implements
return resourcesAvailable;
}
- private boolean tryStartContainer(Container container, boolean force) {
- boolean containerStarted = false;
- // call startContainer without checking available resource when force==true
- if (force || resourceAvailableToStartContainer(
- container)) {
+ /**
+ * Launch all queued GUARANTEED containers without checking resource
+ * availability. This is an optimization in cases where OPPORTUNISTIC
+ * containers are not allowed on the node.
+ */
+ private void forcefullyStartGuaranteedContainers() {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
+ while (cIter.hasNext()) {
+ Container container = cIter.next();
startContainer(container);
- containerStarted = true;
+ cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
}
- return containerStarted;
}
-
/**
- * Check if there is resource available to start a given container
- * immediately. (This can be extended to include overallocated resources)
- * @param container the container to start
- * @return true if container can be launched directly
+ * Try to launch as many OPPORTUNISTIC containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all OPPORTUNISTIC containers are launched
+ * or there is no OPPORTUNISTIC containers to launch
*/
- private boolean resourceAvailableToStartContainer(Container container) {
- return this.utilizationTracker.hasResourcesAvailable(container);
+ private boolean startOpportunisticContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedOpportunisticContainers.values().iterator();
+ boolean resourcesAvailable = true;
+ while (cIter.hasNext() && resourcesAvailable) {
+ Container container = cIter.next();
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
+ cIter.remove();
+ } else {
+ resourcesAvailable = false;
+ }
+ }
+ return resourcesAvailable;
+ }
+
+ private static boolean isResourceAvailable(
+ Resource resource, Container container) {
+ Resource left = Resources.subtract(resource, container.getResource());
+ return left.getMemorySize() >= 0 && left.getVirtualCores() >= 0;
}
private boolean enqueueContainer(Container container) {
@@ -438,6 +529,7 @@ public class ContainerScheduler extends AbstractService implements
boolean isQueued;
if (isGuaranteedContainer) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
isQueued = true;
} else {
if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
@@ -482,18 +574,7 @@ public class ContainerScheduler extends AbstractService implements
// enough number of opportunistic containers.
if (isGuaranteedContainer) {
enqueueContainer(container);
-
- // When opportunistic container not allowed (which is determined by
- // max-queue length of pending opportunistic containers <= 0), start
- // guaranteed containers without looking at available resources.
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
-
- // if the guaranteed container is queued, we need to preempt opportunistic
- // containers for make room for it
- if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
- reclaimOpportunisticContainerResources(container);
- }
+ startPendingContainers(true);
} else {
// Given an opportunistic container, we first try to start as many queuing
// guaranteed containers as possible followed by queuing opportunistic
@@ -511,19 +592,19 @@ public class ContainerScheduler extends AbstractService implements
}
@SuppressWarnings("unchecked")
- private void reclaimOpportunisticContainerResources(Container container) {
+ private void reclaimOpportunisticContainerResources() {
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources(
- container.getContainerId());
- // Kill the opportunistic containers that were chosen.
- for (Container contToReclaim : extraOppContainersToReclaim) {
+ pickOpportunisticContainersToReclaimResources();
+ killOpportunisticContainers(extraOppContainersToReclaim);
+ }
+
+ private void killOpportunisticContainers(
+ Collection<Container> containersToReclaim) {
+ for (Container contToReclaim : containersToReclaim) {
String preemptionAction = usePauseEventForPreemption == true ? "paused" :
- "resumed";
- LOG.info(
- "Container {} will be {} to start the "
- + "execution of guaranteed container {}.",
- contToReclaim.getContainerId(), preemptionAction,
- container.getContainerId());
+ "preempted";
+ LOG.info("Container {} will be {} to start the execution of guaranteed" +
+ " containers.", contToReclaim.getContainerId(), preemptionAction);
if (usePauseEventForPreemption) {
contToReclaim.sendPauseEvent(
@@ -534,6 +615,8 @@ public class ContainerScheduler extends AbstractService implements
"Container Killed to make room for Guaranteed Container.");
}
oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
+ Resources.addTo(
+ opportunisticResourcesToBeReleased, contToReclaim.getResource());
}
}
@@ -542,7 +625,7 @@ public class ContainerScheduler extends AbstractService implements
// Skip to put into runningContainers and addUtilization when recover
if (!runningContainers.containsKey(container.getContainerId())) {
runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.addContainerResources(container);
+ this.utilizationTracker.containerLaunched(container);
}
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -551,14 +634,12 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources(
- ContainerId containerToStartId) {
+ private List<Container> pickOpportunisticContainersToReclaimResources() {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp(
- containerToStartId);
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
@@ -577,15 +658,19 @@ public class ContainerScheduler extends AbstractService implements
continue;
}
extraOpportContainersToKill.add(runningCont);
+ // In the case of over-allocation, the running container may not
+ // release as much resources as it has requested, but we'll check
+ // again if more containers need to be killed/paused when this
+ // container is released.
ContainersMonitor.decreaseResourceUtilization(
getContainersMonitor(), resourcesToFreeUp,
runningCont.getResource());
}
}
if (!hasSufficientResources(resourcesToFreeUp)) {
- LOG.warn("There are no sufficient resources to start guaranteed [{}]" +
- "at the moment. Opportunistic containers are in the process of" +
- "being killed to make room.", containerToStartId);
+ LOG.warn("There are no sufficient resources to start guaranteed" +
+ " containers at the moment. Opportunistic containers are in" +
+ " the process of being killed to make room.");
}
return extraOpportContainersToKill;
}
@@ -600,34 +685,42 @@ public class ContainerScheduler extends AbstractService implements
* getContainersMonitor().getVCoresAllocatedForContainers()) <= 0;
}
- private ResourceUtilization resourcesToFreeUp(
- ContainerId containerToStartId) {
+ /**
+ * Determine how much resources are needed to be freed up to launch the given
+ * GUARANTEED container. Used to determine how many running OPPORTUNISTIC
+ * containers need to be killed/paused, assuming OPPORTUNISTIC containers to
+ * be killed/paused will release the amount of resources they have requested.
+ *
+ * If the node is over-allocating itself, this may cause not enough
+ * OPPORTUNISTIC containers being killed/paused in cases where the running
+ * OPPORTUNISTIC containers are not consuming fully their resource requests.
+ * We'd check again upon container completion events to see if more running
+ * OPPORTUNISTIC containers need to be killed/paused.
+ *
+ * @return the amount of resource needed to be reclaimed for this container
+ */
+ private ResourceUtilization resourcesToFreeUp() {
// Get allocation of currently allocated containers.
ResourceUtilization resourceAllocationToFreeUp = ResourceUtilization
- .newInstance(this.utilizationTracker.getCurrentUtilization());
-
- // Add to the allocation the allocation of the pending guaranteed
- // containers that will start before the current container will be started.
- for (Container container : queuedGuaranteedContainers.values()) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
- if (container.getContainerId().equals(containerToStartId)) {
- break;
- }
- }
+ .newInstance(0, 0, 0.0f);
+
+ // Add to the allocation the allocation of pending guaranteed containers.
+ ContainersMonitor.increaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, guaranteedResourcesDemanded);
// These resources are being freed, likely at the behest of another
// guaranteed container..
- for (Container container : oppContainersToKill.values()) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, opportunisticResourcesToBeReleased);
+
+ // Deduct any remaining resources available
+ Resource availableResources = utilizationTracker.getAvailableResources();
+ if (availableResources.getVirtualCores() > 0 &&
+ availableResources.getMemorySize() > 0) {
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, availableResources);
}
- // Subtract the overall node resources.
- getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
- resourceAllocationToFreeUp);
return resourceAllocationToFreeUp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 294eddf..9ad4f91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -28,5 +28,7 @@ public enum ContainerSchedulerEventType {
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
CONTAINER_PAUSED,
- RECOVERY_COMPLETED
+ RECOVERY_COMPLETED,
+ // Producer: Containers Monitor when over-allocation is on
+ SCHEDULE_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
new file mode 100644
index 0000000..58b73d2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * Keeps track of containers utilization over time and determines how much
+ * resources are available to launch containers when over-allocation is on.
+ */
+public abstract class NMAllocationPolicy {
+ protected final ResourceThresholds overAllocationThresholds;
+ protected final ContainersMonitor containersMonitor;
+
+ public NMAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ this.containersMonitor = containersMonitor;
+ this.overAllocationThresholds = overAllocationThresholds;
+ }
+
+ /**
+ * Handle container launch events.
+ * @param container the container that has been launched
+ */
+ public void containerLaunched(Container container) {
+
+ }
+
+ /**
+ * Handle container release events.
+ * @param container the container that has been released
+ */
+ public void containerReleased(Container container) {
+
+ }
+
+ /**
+ * Get the amount of resources to launch containers when
+ * over-allocation is turned on.
+ * @return the amount of resources available to launch containers
+ */
+ public abstract Resource getAvailableResources();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 3c17eca..98d99c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -38,22 +39,20 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
- * Add Container's resources to Node Utilization.
- * @param container Container.
+ * Get the amount of resources currently available to launch containers.
+ * @return Resource resources available to launch containers
*/
- void addContainerResources(Container container);
+ Resource getAvailableResources();
/**
- * Subtract Container's resources to Node Utilization.
+ * Add Container's resources to Node Utilization upon container launch.
* @param container Container.
*/
- void subtractContainerResource(Container container);
+ void containerLaunched(Container container);
/**
- * Check if NM has resources available currently to run the container.
+ * Subtract Container's resources to Node Utilization upon container release.
* @param container Container.
- * @return True, if NM has resources available currently to run the container.
*/
- boolean hasResourcesAvailable(Container container);
-
+ void containerReleased(Container container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
new file mode 100644
index 0000000..f486506
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * An implementation of NMAllocationPolicy based on the
+ * snapshot of the latest containers utilization to determine how much
+ * resources are available * to launch containers when over-allocation
+ * is turned on.
+ */
+public class SnapshotBasedOverAllocationPolicy
+ extends NMAllocationPolicy {
+
+ public SnapshotBasedOverAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ super(overAllocationThresholds, containersMonitor);
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ ResourceUtilization utilization =
+ containersMonitor.getContainersUtilization(true).getUtilization();
+ long memoryAvailable = Math.round(
+ overAllocationThresholds.getMemoryThreshold() *
+ containersMonitor.getPmemAllocatedForContainers()) -
+ (utilization.getPhysicalMemory() << 20);
+ int vcoreAvailable = Math.round(
+ (overAllocationThresholds.getCpuThreshold() - utilization.getCPU()) *
+ containersMonitor.getVCoresAllocatedForContainers());
+ return Resource.newInstance(memoryAvailable >> 20, vcoreAvailable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
new file mode 100644
index 0000000..6f9bc82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+* An resource availability tracker that determines if there are resources
+* available based on if there are unallocated resources or if there are
+* un-utilized resources.
+*/
+public class UtilizationBasedResourceTracker
+ extends AllocationBasedResourceTracker {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private final NMAllocationPolicy overAllocationPolicy;
+
+ UtilizationBasedResourceTracker(ContainerScheduler scheduler) {
+ super(scheduler);
+ this.overAllocationPolicy =
+ getContainersMonitor().getContainerOverAllocationPolicy();
+ }
+
+ @Override
+ public void containerLaunched(Container container) {
+ super.containerLaunched(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerLaunched(container);
+ }
+ }
+
+ @Override
+ public void containerReleased(Container container) {
+ super.containerReleased(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerReleased(container);
+ }
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ Resource resourceBasedOnAllocation = getUnallocatedResources();
+ Resource resourceBasedOnUtilization =
+ getResourcesAvailableBasedOnUtilization();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The amount of resources available based on allocation is " +
+ resourceBasedOnAllocation + ", based on utilization is " +
+ resourceBasedOnUtilization);
+ }
+
+ return Resources.componentwiseMax(resourceBasedOnAllocation,
+ resourceBasedOnUtilization);
+ }
+
+ /**
+ * Get the amount of resources based on the slack between
+ * the actual utilization and desired utilization.
+ * @return Resource resource available
+ */
+ private Resource getResourcesAvailableBasedOnUtilization() {
+ if (overAllocationPolicy == null) {
+ return Resources.none();
+ }
+
+ return overAllocationPolicy.getAvailableResources();
+ }
+
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return getContainersMonitor().getContainersUtilization(false)
+ .getUtilization();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 493aa4c..92613ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -345,6 +346,40 @@ public abstract class BaseContainerManagerTest {
fStates.contains(containerStatus.getState()));
}
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ ContainerSubState finalState)
+ throws InterruptedException, YarnException, IOException {
+ waitForContainerSubState(containerManager, containerID,
+ Arrays.asList(finalState), 20);
+ }
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ List<ContainerSubState> finalStates, int timeOutMax)
+ throws InterruptedException, YarnException, IOException {
+ List<ContainerId> list = new ArrayList<>();
+ list.add(containerID);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(list);
+ ContainerStatus containerStatus;
+ HashSet<ContainerSubState> fStates = new HashSet<>(finalStates);
+ int timeoutSecs = 0;
+ do {
+ Thread.sleep(1000);
+ containerStatus =
+ containerManager.getContainerStatuses(request)
+ .getContainerStatuses().get(0);
+ LOG.info("Waiting for container to get into one of states " + fStates
+ + ". Current state is " + containerStatus.getContainerSubState());
+ timeoutSecs += 1;
+ } while (!fStates.contains(containerStatus.getContainerSubState())
+ && timeoutSecs < timeOutMax);
+ LOG.info("Container state is " + containerStatus.getContainerSubState());
+ Assert.assertTrue("ContainerSubState is not correct (timedout)",
+ fStates.contains(containerStatus.getContainerSubState()));
+ }
+
+
public static void waitForApplicationState(
ContainerManagerImpl containerManager, ApplicationId appID,
ApplicationState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 8aee532..c071283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -288,7 +288,7 @@ public class TestContainersMonitorResourceChange {
// will be 0.
assertEquals(
"Resource utilization must be default with MonitorThread's first run",
- 0, containersMonitor.getContainersUtilization()
+ 0, containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
// Verify the container utilization value. Since atleast one round is done,
@@ -303,8 +303,9 @@ public class TestContainersMonitorResourceChange {
ContainersMonitorImpl containersMonitor, int timeoutMsecs)
throws InterruptedException {
int timeWaiting = 0;
- while (0 == containersMonitor.getContainersUtilization()
- .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) {
+ while (0 == containersMonitor.getContainersUtilization(false)
+ .getUtilization().compareTo(
+ ResourceUtilization.newInstance(0, 0, 0.0f))) {
if (timeWaiting >= timeoutMsecs) {
break;
}
@@ -316,7 +317,7 @@ public class TestContainersMonitorResourceChange {
}
assertTrue("Resource utilization is not changed from second run onwards",
- 0 != containersMonitor.getContainersUtilization()
+ 0 != containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
new file mode 100644
index 0000000..1e8bfdf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link AllocationBasedResourceTracker} class.
+ */
+public class TestAllocationBasedResourceTracker {
+
+ private ContainerScheduler mockContainerScheduler;
+
+ @Before
+ public void setup() {
+ mockContainerScheduler = mock(ContainerScheduler.class);
+ ContainersMonitor containersMonitor =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
+ conf.setInt(YarnConfiguration.NM_VCORES, 8);
+ containersMonitor.init(conf);
+ when(mockContainerScheduler.getContainersMonitor())
+ .thenReturn(containersMonitor);
+ }
+
+ /**
+ * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
+ * hasResourceAvailable should return false.
+ */
+ @Test
+ public void testHasResourcesAvailable() {
+ AllocationBasedResourceTracker tracker =
+ new AllocationBasedResourceTracker(mockContainerScheduler);
+ Container testContainer = mock(Container.class);
+ when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
+ for (int i = 0; i < 2; i++) {
+ Assert.assertTrue(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ tracker.containerLaunched(testContainer);
+ }
+ Assert.assertFalse(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ }
+
+ private static boolean isResourcesAvailable(
+ Resource available, Container container) {
+ return available.compareTo(container.getResource()) >= 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 82c2147..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for the {@link AllocationBasedResourceUtilizationTracker} class.
- */
-public class TestAllocationBasedResourceUtilizationTracker {
-
- private ContainerScheduler mockContainerScheduler;
-
- @Before
- public void setup() {
- mockContainerScheduler = mock(ContainerScheduler.class);
- ContainersMonitor containersMonitor =
- new ContainersMonitorImpl(mock(ContainerExecutor.class),
- mock(AsyncDispatcher.class), mock(Context.class));
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
- conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
- conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
- conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
- conf.setInt(YarnConfiguration.NM_VCORES, 8);
- containersMonitor.init(conf);
- when(mockContainerScheduler.getContainersMonitor())
- .thenReturn(containersMonitor);
- }
-
- /**
- * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
- * hasResourceAvailable should return false.
- */
- @Test
- public void testHasResourcesAvailable() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- Container testContainer = mock(Container.class);
- when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
- for (int i = 0; i < 2; i++) {
- Assert.assertTrue(tracker.hasResourcesAvailable(testContainer));
- tracker.addContainerResources(testContainer);
- }
- Assert.assertFalse(tracker.hasResourcesAvailable(testContainer));
- }
-
- /**
- * Test the case where the current allocation has been truncated to 0.8888891
- * (8/9 cores used). Request 1 additional core - hasEnoughCpu should return
- * true.
- */
- @Test
- public void testHasEnoughCpu() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- float currentAllocation = 0.8888891f;
- long totalCores = 9;
- int alreadyUsedCores = 8;
- Assert.assertTrue(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores));
- Assert.assertFalse(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores + 1));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-15736. Trash : Negative
Value For Deletion Interval Leads To Abnormal Behaviour. Contributed by Ayush
Saxena.
Posted by ha...@apache.org.
HADOOP-15736. Trash : Negative Value For Deletion Interval Leads To Abnormal Behaviour. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ad27e97
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ad27e97
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ad27e97
Branch: refs/heads/YARN-1011
Commit: 7ad27e97f05b13b33fdcef9cb63ace9c1728bfb5
Parents: 6fc293f
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Sep 20 09:31:35 2018 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Sep 20 09:31:35 2018 +0530
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/TrashPolicyDefault.java | 8 +++++++-
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 6 ++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad27e97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 6e101a2..39d5e73 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -101,6 +101,12 @@ public class TrashPolicyDefault extends TrashPolicy {
this.emptierInterval = (long)(conf.getFloat(
FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
* MSECS_PER_MINUTE);
+ if (deletionInterval < 0) {
+ LOG.warn("Invalid value {} for deletion interval,"
+ + " deletion interaval can not be negative."
+ + "Changing to default value 0", deletionInterval);
+ this.deletionInterval = 0;
+ }
}
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
@@ -109,7 +115,7 @@ public class TrashPolicyDefault extends TrashPolicy {
@Override
public boolean isEnabled() {
- return deletionInterval != 0;
+ return deletionInterval > 0;
}
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad27e97/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 568821b..04f56fb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -132,6 +132,9 @@ public class TestTrash {
conf.setLong(FS_TRASH_INTERVAL_KEY, 0); // disabled
assertFalse(new Trash(conf).isEnabled());
+ conf.setLong(FS_TRASH_INTERVAL_KEY, -1); // disabled
+ assertFalse(new Trash(conf).isEnabled());
+
conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute
assertTrue(new Trash(conf).isEnabled());
@@ -526,6 +529,9 @@ public class TestTrash {
conf.setLong(FS_TRASH_INTERVAL_KEY, 0); // disabled
assertFalse(new Trash(conf).isEnabled());
+ conf.setLong(FS_TRASH_INTERVAL_KEY, -1); // disabled
+ assertFalse(new Trash(conf).isEnabled());
+
conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute
assertTrue(new Trash(conf).isEnabled());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-4511. Common scheduler
changes to support scheduler-specific oversubscription implementations.
Posted by ha...@apache.org.
YARN-4511. Common scheduler changes to support scheduler-specific oversubscription implementations.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce4c4a70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce4c4a70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce4c4a70
Branch: refs/heads/YARN-1011
Commit: ce4c4a70839a66f80081ff5295266759b466a158
Parents: d7cdf2c
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Nov 2 09:12:19 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 14:56:07 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 6 +
.../resourcemanager/ResourceTrackerService.java | 3 +-
.../monitor/capacity/TempSchedulerNode.java | 2 +-
.../server/resourcemanager/rmnode/RMNode.java | 7 +
.../resourcemanager/rmnode/RMNodeImpl.java | 13 +-
.../scheduler/AbstractYarnScheduler.java | 4 +-
.../scheduler/ClusterNodeTracker.java | 6 +-
.../scheduler/SchedulerNode.java | 323 +++++++++++----
.../scheduler/SchedulerNodeReport.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../allocator/RegularContainerAllocator.java | 4 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../common/fica/FiCaSchedulerNode.java | 11 +-
.../scheduler/fair/FSPreemptionThread.java | 2 +-
.../scheduler/fair/FSSchedulerNode.java | 9 +-
.../yarn/server/resourcemanager/MockNodes.java | 6 +
.../TestWorkPreservingRMRestart.java | 39 +-
...alCapacityPreemptionPolicyMockFramework.java | 2 +-
...alCapacityPreemptionPolicyMockFramework.java | 6 +-
.../scheduler/TestAbstractYarnScheduler.java | 6 +-
.../scheduler/TestSchedulerNode.java | 393 +++++++++++++++++++
.../capacity/TestCapacityScheduler.java | 2 +-
.../TestCapacitySchedulerAsyncScheduling.java | 10 +-
.../scheduler/capacity/TestLeafQueue.java | 4 +-
.../TestNodeLabelContainerAllocation.java | 14 +-
.../fair/TestContinuousScheduling.java | 42 +-
.../scheduler/fair/TestFSSchedulerNode.java | 18 +-
.../scheduler/fair/TestFairScheduler.java | 14 +-
.../scheduler/fifo/TestFifoScheduler.java | 4 +-
30 files changed, 795 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 2eee351..350f4a3 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -203,6 +204,11 @@ public class NodeInfo {
}
@Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
+ @Override
public long getUntrackedTimeStamp() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index c73fb15..bb6fb9d 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -190,6 +191,11 @@ public class RMNodeWrapper implements RMNode {
}
@Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return node.getOverAllocationInfo();
+ }
+
+ @Override
public long getUntrackedTimeStamp() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index b67172e..2edc050 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -397,7 +397,8 @@ public class ResourceTrackerService extends AbstractService implements
.getCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
- resolve(host), capability, nodeManagerVersion, physicalResource);
+ resolve(host), capability, nodeManagerVersion, physicalResource,
+ request.getOverAllocationInfo());
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
if (oldNode == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
index 320f262..25777af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempSchedulerNode.java
@@ -51,7 +51,7 @@ public class TempSchedulerNode {
public static TempSchedulerNode fromSchedulerNode(
FiCaSchedulerNode schedulerNode) {
TempSchedulerNode n = new TempSchedulerNode();
- n.totalResource = Resources.clone(schedulerNode.getTotalResource());
+ n.totalResource = Resources.clone(schedulerNode.getCapacity());
n.allocatedResource = Resources.clone(schedulerNode.getAllocatedResource());
n.runningContainers = schedulerNode.getCopiedListOfRunningContainers();
n.reservedContainer = schedulerNode.getReservedContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index c77d29c..aa19483 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
/**
* Node managers information on available resources
@@ -117,6 +118,12 @@ public interface RMNode {
public ResourceUtilization getNodeUtilization();
/**
+ * Get the node overallocation threshold.
+ * @return the overallocation threshold
+ */
+ OverAllocationInfo getOverAllocationInfo();
+
+ /**
* the physical resources in the node.
* @return the physical resources in the node.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 65a0c20..b53b32b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType;
@@ -112,6 +113,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private final WriteLock writeLock;
private final ConcurrentLinkedQueue<UpdatedContainerInfo> nodeUpdateQueue;
+ private final OverAllocationInfo overallocationInfo;
private volatile boolean nextHeartBeat = true;
private final NodeId nodeId;
@@ -367,12 +369,13 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
int cmPort, int httpPort, Node node, Resource capability,
String nodeManagerVersion) {
this(nodeId, context, hostName, cmPort, httpPort, node, capability,
- nodeManagerVersion, null);
+ nodeManagerVersion, null, null);
}
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
int cmPort, int httpPort, Node node, Resource capability,
- String nodeManagerVersion, Resource physResource) {
+ String nodeManagerVersion, Resource physResource,
+ OverAllocationInfo overAllocationInfo) {
this.nodeId = nodeId;
this.context = context;
this.hostName = hostName;
@@ -387,6 +390,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.nodeManagerVersion = nodeManagerVersion;
this.timeStamp = 0;
this.physicalResource = physResource;
+ this.overallocationInfo = overAllocationInfo;
this.latestNodeHeartBeatResponse.setResponseId(0);
@@ -536,6 +540,11 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
}
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return this.overallocationInfo;
+ }
+
public void setNodeUtilization(ResourceUtilization nodeUtilization) {
this.writeLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 9d2b058..6b1fdcb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -357,7 +357,7 @@ public abstract class AbstractYarnScheduler
}
application.containerLaunchedOnNode(containerId, node.getNodeID());
- node.containerStarted(containerId);
+ node.containerLaunched(containerId);
} finally {
readLock.unlock();
}
@@ -825,7 +825,7 @@ public abstract class AbstractYarnScheduler
writeLock.lock();
SchedulerNode node = getSchedulerNode(nm.getNodeID());
Resource newResource = resourceOption.getResource();
- Resource oldResource = node.getTotalResource();
+ Resource oldResource = node.getCapacity();
if (!oldResource.equals(newResource)) {
// Notify NodeLabelsManager about this change
rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 8c7e447..90b1c3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -102,7 +102,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
nodesList.add(node);
// Update cluster capacity
- Resources.addTo(clusterCapacity, node.getTotalResource());
+ Resources.addTo(clusterCapacity, node.getCapacity());
staleClusterCapacity = Resources.clone(clusterCapacity);
// Update maximumAllocation
@@ -197,7 +197,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
// Update cluster capacity
- Resources.subtractFrom(clusterCapacity, node.getTotalResource());
+ Resources.subtractFrom(clusterCapacity, node.getCapacity());
staleClusterCapacity = Resources.clone(clusterCapacity);
// Update maximumAllocation
@@ -259,7 +259,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
private void updateMaxResources(SchedulerNode node, boolean add) {
- Resource totalResource = node.getTotalResource();
+ Resource totalResource = node.getCapacity();
ResourceInformation[] totalResources;
if (totalResource != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index b35aeba..8f5ae0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -60,24 +61,33 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
+ private Resource capacity;
private Resource unallocatedResource = Resource.newInstance(0, 0);
- private Resource allocatedResource = Resource.newInstance(0, 0);
- private Resource totalResource;
+
private RMContainer reservedContainer;
- private volatile int numContainers;
private volatile ResourceUtilization containersUtilization =
ResourceUtilization.newInstance(0, 0, 0f);
private volatile ResourceUtilization nodeUtilization =
ResourceUtilization.newInstance(0, 0, 0f);
- /* set of containers that are allocated containers */
- private final Map<ContainerId, ContainerInfo> launchedContainers =
- new HashMap<>();
+ private final Map<ContainerId, ContainerInfo>
+ allocatedContainers = new HashMap<>();
+
+ private volatile int numGuaranteedContainers = 0;
+ private Resource allocatedResourceGuaranteed = Resource.newInstance(0, 0);
+
+ private volatile int numOpportunisticContainers = 0;
+ private Resource allocatedResourceOpportunistic = Resource.newInstance(0, 0);
private final RMNode rmNode;
private final String nodeName;
private final RMContext rmContext;
+ // The total amount of resources requested by containers that have been
+ // allocated but not yet launched on the node.
+ protected Resource resourceAllocatedPendingLaunch =
+ Resource.newInstance(0, 0);
+
private volatile Set<String> labels = null;
private volatile Set<NodeAttribute> nodeAttributes = null;
@@ -90,7 +100,7 @@ public abstract class SchedulerNode {
this.rmNode = node;
this.rmContext = node.getRMContext();
this.unallocatedResource = Resources.clone(node.getTotalCapability());
- this.totalResource = Resources.clone(node.getTotalCapability());
+ this.capacity = Resources.clone(node.getTotalCapability());
if (usePortForNodeName) {
nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
} else {
@@ -113,9 +123,9 @@ public abstract class SchedulerNode {
* @param resource Total resources on the node.
*/
public synchronized void updateTotalResource(Resource resource){
- this.totalResource = resource;
- this.unallocatedResource = Resources.subtract(totalResource,
- this.allocatedResource);
+ this.capacity = Resources.clone(resource);
+ this.unallocatedResource = Resources.subtract(capacity,
+ this.allocatedResourceGuaranteed);
}
/**
@@ -174,17 +184,83 @@ public abstract class SchedulerNode {
protected synchronized void allocateContainer(RMContainer rmContainer,
boolean launchedOnNode) {
Container container = rmContainer.getContainer();
- if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
- deductUnallocatedResource(container.getResource());
- ++numContainers;
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ guaranteedContainerResourceAllocated(rmContainer, launchedOnNode);
+ } else {
+ opportunisticContainerResourceAllocated(rmContainer, launchedOnNode);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigned container " + container.getId() + " of capacity "
+ + container.getResource() + " and type " +
+ container.getExecutionType() + " on host " + toString());
+ }
+ }
+
+ /**
+ * Handle an allocation of a GUARANTEED container.
+ * @param rmContainer the allocated GUARANTEED container
+ * @param launchedOnNode true if the container has been launched
+ */
+ private void guaranteedContainerResourceAllocated(
+ RMContainer rmContainer, boolean launchedOnNode) {
+ Container container = rmContainer.getContainer();
+
+ if (container.getExecutionType() != ExecutionType.GUARANTEED) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ allocatedContainers.put(container.getId(),
+ new ContainerInfo(rmContainer, launchedOnNode));
+
+ Resource resource = container.getResource();
+ if (containerResourceAllocated(resource, allocatedResourceGuaranteed)) {
+ Resources.subtractFrom(unallocatedResource, resource);
+ }
+
+ numGuaranteedContainers++;
+ }
+
+ /**
+ * Handle an allocation of a OPPORTUNISTIC container.
+ * @param rmContainer the allocated OPPORTUNISTIC container
+ * @param launchedOnNode true if the container has been launched
+ */
+ private void opportunisticContainerResourceAllocated(
+ RMContainer rmContainer, boolean launchedOnNode) {
+ Container container = rmContainer.getContainer();
+
+ if (container.getExecutionType() != ExecutionType.OPPORTUNISTIC) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
}
- launchedContainers.put(container.getId(),
+ allocatedContainers.put(rmContainer.getContainerId(),
new ContainerInfo(rmContainer, launchedOnNode));
+ if (containerResourceAllocated(
+ container.getResource(), allocatedResourceOpportunistic)) {
+ // nothing to do here
+ }
+ numOpportunisticContainers++;
+ }
+
+ private boolean containerResourceAllocated(Resource allocated,
+ Resource aggregatedResources) {
+ if (allocated == null) {
+ LOG.error("Invalid deduction of null resource for "
+ + rmNode.getNodeAddress());
+ return false;
+ }
+ Resources.addTo(resourceAllocatedPendingLaunch, allocated);
+ Resources.addTo(aggregatedResources, allocated);
+ return true;
}
+
/**
- * Get unallocated resources on the node.
+ * Get resources that are not allocated to GUARANTEED containers on the node.
* @return Unallocated resources on the node
*/
public synchronized Resource getUnallocatedResource() {
@@ -192,42 +268,57 @@ public abstract class SchedulerNode {
}
/**
- * Get allocated resources on the node.
- * @return Allocated resources on the node
+ * Get resources allocated to GUARANTEED containers on the node.
+ * @return Allocated resources to GUARANTEED containers on the node
*/
public synchronized Resource getAllocatedResource() {
- return this.allocatedResource;
+ return this.allocatedResourceGuaranteed;
+ }
+
+ /**
+ * Get resources allocated to OPPORTUNISTIC containers on the node.
+ * @return Allocated resources to OPPORTUNISTIC containers on the node
+ */
+ public synchronized Resource getOpportunisticResourceAllocated() {
+ return this.allocatedResourceOpportunistic;
+ }
+
+ @VisibleForTesting
+ public synchronized Resource getResourceAllocatedPendingLaunch() {
+ return this.resourceAllocatedPendingLaunch;
}
/**
* Get total resources on the node.
* @return Total resources on the node.
*/
- public synchronized Resource getTotalResource() {
- return this.totalResource;
+ public synchronized Resource getCapacity() {
+ return this.capacity;
}
/**
- * Check if a container is launched by this node.
+ * Check if a GUARANTEED container is launched by this node.
* @return If the container is launched by the node.
*/
- public synchronized boolean isValidContainer(ContainerId containerId) {
- if (launchedContainers.containsKey(containerId)) {
- return true;
- }
- return false;
+ @VisibleForTesting
+ public synchronized boolean isValidGuaranteedContainer(
+ ContainerId containerId) {
+ ContainerInfo containerInfo = allocatedContainers.get(containerId);
+ return containerInfo != null && ExecutionType.GUARANTEED ==
+ containerInfo.container.getExecutionType();
}
/**
- * Update the resources of the node when releasing a container.
- * @param container Container to release.
+ * Check if an OPPORTUNISTIC container is launched by this node.
+ * @param containerId id of the container to check
+ * @return If the container is launched by the node.
*/
- protected synchronized void updateResourceForReleasedContainer(
- Container container) {
- if (container.getExecutionType() == ExecutionType.GUARANTEED) {
- addUnallocatedResource(container.getResource());
- --numContainers;
- }
+ @VisibleForTesting
+ public synchronized boolean isValidOpportunisticContainer(
+ ContainerId containerId) {
+ ContainerInfo containerInfo = allocatedContainers.get(containerId);
+ return containerInfo != null && ExecutionType.OPPORTUNISTIC ==
+ containerInfo.container.getExecutionType();
}
/**
@@ -237,17 +328,30 @@ public abstract class SchedulerNode {
*/
public synchronized void releaseContainer(ContainerId containerId,
boolean releasedByNode) {
- ContainerInfo info = launchedContainers.get(containerId);
- if (info == null) {
+ RMContainer rmContainer = getContainer(containerId);
+ if (rmContainer == null) {
+ LOG.warn("Invalid container " + containerId + " is released.");
+ return;
+ }
+
+ if (!allocatedContainers.containsKey(containerId)) {
+ // do not process if the container is never allocated on the node
return;
}
- if (!releasedByNode && info.launchedOnNode) {
- // wait until node reports container has completed
+
+ if (!releasedByNode &&
+ allocatedContainers.get(containerId).launchedOnNode) {
+ // only process if the container has not been launched on a node
+ // yet or it is released by node.
return;
}
- launchedContainers.remove(containerId);
- Container container = info.container.getContainer();
+ Container container = rmContainer.getContainer();
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ guaranteedContainerReleased(container);
+ } else {
+ opportunisticContainerReleased(container);
+ }
// We remove allocation tags when a container is actually
// released on NM. This is to avoid running into situation
@@ -260,14 +364,16 @@ public abstract class SchedulerNode {
container.getId(), container.getAllocationTags());
}
- updateResourceForReleasedContainer(container);
-
if (LOG.isDebugEnabled()) {
LOG.debug("Released container " + container.getId() + " of capacity "
- + container.getResource() + " on host " + rmNode.getNodeAddress()
- + ", which currently has " + numContainers + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available" + ", release resources=" + true);
+ + container.getResource() + " on host " + rmNode.getNodeAddress()
+ + ", with " + numGuaranteedContainers
+ + " guaranteed containers taking"
+ + getAllocatedResource() + " and " + numOpportunisticContainers
+ + " opportunistic containers taking "
+ + getOpportunisticResourceAllocated()
+ + " and " + getUnallocatedResource() + " (guaranteed) available"
+ + ", release resources=" + true);
}
}
@@ -275,42 +381,75 @@ public abstract class SchedulerNode {
* Inform the node that a container has launched.
* @param containerId ID of the launched container
*/
- public synchronized void containerStarted(ContainerId containerId) {
- ContainerInfo info = launchedContainers.get(containerId);
- if (info != null) {
+ public synchronized void containerLaunched(ContainerId containerId) {
+ ContainerInfo info = allocatedContainers.get(containerId);
+ if (info != null && !info.launchedOnNode) {
info.launchedOnNode = true;
+ Resources.subtractFrom(resourceAllocatedPendingLaunch,
+ info.container.getContainer().getResource());
}
}
/**
- * Add unallocated resources to the node. This is used when unallocating a
- * container.
- * @param resource Resources to add.
+ * Handle the release of a GUARANTEED container.
+ * @param container Container to release.
*/
- private synchronized void addUnallocatedResource(Resource resource) {
- if (resource == null) {
- LOG.error("Invalid resource addition of null resource for "
- + rmNode.getNodeAddress());
- return;
+ protected synchronized void guaranteedContainerReleased(
+ Container container) {
+ if (container.getExecutionType() != ExecutionType.GUARANTEED) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ if (containerResourceReleased(container, allocatedResourceGuaranteed)) {
+ Resources.addTo(unallocatedResource, container.getResource());
}
- Resources.addTo(unallocatedResource, resource);
- Resources.subtractFrom(allocatedResource, resource);
+ // do not update allocated containers until the resources of
+ // the container are released because we need to check if we
+ // need to update resourceAllocatedPendingLaunch in case the
+ // container has not been launched on the node.
+ allocatedContainers.remove(container.getId());
+ numGuaranteedContainers--;
}
/**
- * Deduct unallocated resources from the node. This is used when allocating a
- * container.
- * @param resource Resources to deduct.
+ * Handle the release of an OPPORTUNISTIC container.
+ * @param container Container to release.
*/
- @VisibleForTesting
- public synchronized void deductUnallocatedResource(Resource resource) {
- if (resource == null) {
- LOG.error("Invalid deduction of null resource for "
+ private void opportunisticContainerReleased(
+ Container container) {
+ if (container.getExecutionType() != ExecutionType.OPPORTUNISTIC) {
+ throw new YarnRuntimeException("Inapplicable ExecutionType: " +
+ container.getExecutionType());
+ }
+
+ if (containerResourceReleased(container, allocatedResourceOpportunistic)) {
+ // nothing to do here
+ }
+ // do not update allocated containers until the resources of
+ // the container are released because we need to check if we
+ // need to update resourceAllocatedPendingLaunch in case the
+ // container has not been launched on the node.
+ allocatedContainers.remove(container.getId());
+ numOpportunisticContainers--;
+ }
+
+ private boolean containerResourceReleased(Container container,
+ Resource aggregatedResource) {
+ Resource released = container.getResource();
+ if (released == null) {
+ LOG.error("Invalid resource addition of null resource for "
+ rmNode.getNodeAddress());
- return;
+ return false;
}
- Resources.subtractFrom(unallocatedResource, resource);
- Resources.addTo(allocatedResource, resource);
+ Resources.subtractFrom(aggregatedResource, released);
+
+ if (!allocatedContainers.get(container.getId()).launchedOnNode) {
+ // update resourceAllocatedPendingLaunch if the container is has
+ // not yet been launched on the node
+ Resources.subtractFrom(resourceAllocatedPendingLaunch, released);
+ }
+ return true;
}
/**
@@ -330,17 +469,28 @@ public abstract class SchedulerNode {
@Override
public String toString() {
- return "host: " + rmNode.getNodeAddress() + " #containers="
- + getNumContainers() + " available=" + getUnallocatedResource()
- + " used=" + getAllocatedResource();
+ return "host: " + rmNode.getNodeAddress() + " #guaranteed containers=" +
+ getNumGuaranteedContainers() + " #opportunistic containers=" +
+ getNumOpportunisticContainers() + " available=" +
+ getUnallocatedResource() + " used by guaranteed containers=" +
+ allocatedResourceGuaranteed + " used by opportunistic containers=" +
+ allocatedResourceOpportunistic;
+ }
+
+ /**
+ * Get number of active GUARANTEED containers on the node.
+ * @return Number of active OPPORTUNISTIC containers on the node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numGuaranteedContainers;
}
/**
- * Get number of active containers on the node.
- * @return Number of active containers on the node.
+ * Get number of active OPPORTUNISTIC containers on the node.
+ * @return Number of active OPPORTUNISTIC containers on the node.
*/
- public int getNumContainers() {
- return numContainers;
+ public int getNumOpportunisticContainers() {
+ return numOpportunisticContainers;
}
/**
@@ -348,8 +498,8 @@ public abstract class SchedulerNode {
* @return A copy of containers running on the node.
*/
public synchronized List<RMContainer> getCopiedListOfRunningContainers() {
- List<RMContainer> result = new ArrayList<>(launchedContainers.size());
- for (ContainerInfo info : launchedContainers.values()) {
+ List<RMContainer> result = new ArrayList<>(allocatedContainers.size());
+ for (ContainerInfo info : allocatedContainers.values()) {
result.add(info.container);
}
return result;
@@ -359,12 +509,14 @@ public abstract class SchedulerNode {
* Get the containers running on the node with AM containers at the end.
* @return A copy of running containers with AM containers at the end.
*/
- public synchronized List<RMContainer> getRunningContainersWithAMsAtTheEnd() {
+ public synchronized List<RMContainer>
+ getRunningGuaranteedContainersWithAMsAtTheEnd() {
LinkedList<RMContainer> result = new LinkedList<>();
- for (ContainerInfo info : launchedContainers.values()) {
+ for (ContainerInfo info : allocatedContainers.values()) {
if(info.container.isAMContainer()) {
result.addLast(info.container);
- } else {
+ } else if (info.container.getExecutionType() ==
+ ExecutionType.GUARANTEED){
result.addFirst(info.container);
}
}
@@ -377,12 +529,9 @@ public abstract class SchedulerNode {
* @return The container for the specified container ID
*/
protected synchronized RMContainer getContainer(ContainerId containerId) {
- RMContainer container = null;
- ContainerInfo info = launchedContainers.get(containerId);
- if (info != null) {
- container = info.container;
- }
- return container;
+ ContainerInfo info = allocatedContainers.get(containerId);
+
+ return info != null ? info.container : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index fa71a25..ea30d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -31,11 +31,11 @@ public class SchedulerNodeReport {
private final Resource used;
private final Resource avail;
private final int num;
-
+
public SchedulerNodeReport(SchedulerNode node) {
this.used = node.getAllocatedResource();
this.avail = node.getUnallocatedResource();
- this.num = node.getNumContainers();
+ this.num = node.getNumGuaranteedContainers();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 4b274df..0b7115b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1986,7 +1986,7 @@ public class CapacityScheduler extends
// update this node to node label manager
if (labelManager != null) {
labelManager.activateNode(nodeManager.getNodeID(),
- schedulerNode.getTotalResource());
+ schedulerNode.getCapacity());
}
// recover attributes from store if any.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index c0a11a0..5be2507 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -514,13 +514,13 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
Resource capability = pendingAsk.getPerAllocationResource();
Resource available = node.getUnallocatedResource();
- Resource totalResource = node.getTotalResource();
+ Resource totalResource = node.getCapacity();
if (!Resources.lessThanOrEqual(rc, clusterResource,
capability, totalResource)) {
LOG.warn("Node : " + node.getNodeID()
+ " does not have sufficient resource for ask : " + pendingAsk
- + " node total capability : " + node.getTotalResource());
+ + " node total capability : " + node.getCapacity());
// Skip this locality request
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, priority,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 4bfdae9..e6a71f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1072,7 +1072,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
diagnosticMessageBldr.append(" ( Partition : ");
diagnosticMessageBldr.append(node.getLabels());
diagnosticMessageBldr.append(", Total resource : ");
- diagnosticMessageBldr.append(node.getTotalResource());
+ diagnosticMessageBldr.append(node.getCapacity());
diagnosticMessageBldr.append(", Available resource : ");
diagnosticMessageBldr.append(node.getUnallocatedResource());
diagnosticMessageBldr.append(" ).");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 7277779..52165a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -144,9 +144,9 @@ public class FiCaSchedulerNode extends SchedulerNode {
}
@Override
- protected synchronized void updateResourceForReleasedContainer(
+ protected synchronized void guaranteedContainerReleased(
Container container) {
- super.updateResourceForReleasedContainer(container);
+ super.guaranteedContainerReleased(container);
if (killableContainers.containsKey(container.getId())) {
Resources.subtractFrom(totalKillableResources, container.getResource());
killableContainers.remove(container.getId());
@@ -168,9 +168,10 @@ public class FiCaSchedulerNode extends SchedulerNode {
final Container container = rmContainer.getContainer();
LOG.info("Assigned container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + getRMNode().getNodeAddress()
- + ", which has " + getNumContainers() + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available after allocation");
+ + ", which has " + getNumGuaranteedContainers() + " guaranteed"
+ + " containers using " + getAllocatedResource() + ", "
+ + getNumOpportunisticContainers() + " opportunistic containers"
+ + " using " + getOpportunisticResourceAllocated());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index c32565f..dcb076f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -193,7 +193,7 @@ class FSPreemptionThread extends Thread {
// Figure out list of containers to consider
List<RMContainer> containersToCheck =
- node.getRunningContainersWithAMsAtTheEnd();
+ node.getRunningGuaranteedContainersWithAMsAtTheEnd();
containersToCheck.removeAll(node.getContainersForPreemption());
// Initialize potential with unallocated but not reserved resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 44ec9c3..95490f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -242,11 +242,12 @@ public class FSSchedulerNode extends SchedulerNode {
super.allocateContainer(rmContainer, launchedOnNode);
if (LOG.isDebugEnabled()) {
final Container container = rmContainer.getContainer();
- LOG.debug("Assigned container " + container.getId() + " of capacity "
+ LOG.info("Assigned container " + container.getId() + " of capacity "
+ container.getResource() + " on host " + getRMNode().getNodeAddress()
- + ", which has " + getNumContainers() + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available after allocation");
+ + ", which has " + getNumGuaranteedContainers() + " guaranteed "
+ + "containers using " + getAllocatedResource() + ", "
+ + getNumOpportunisticContainers() + " opportunistic containers "
+ + "using " + getOpportunisticResourceAllocated());
}
Resource allocated = rmContainer.getAllocatedResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index a871993..ce1d6b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
@@ -263,6 +264,11 @@ public class MockNodes {
return this.nodeUtilization;
}
+ @Override
+ public OverAllocationInfo getOverAllocationInfo() {
+ return null;
+ }
+
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
return OpportunisticContainersStatus.newInstance();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index a821b0a..df409f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -232,13 +232,14 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
Resource nmResource =
Resource.newInstance(nm1.getMemory(), nm1.getvCores());
- assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
- assertTrue(schedulerNode1.isValidContainer(runningContainer
- .getContainerId()));
- assertFalse(schedulerNode1.isValidContainer(completedContainer
- .getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ amContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ runningContainer.getContainerId()));
+ assertFalse(schedulerNode1.isValidGuaranteedContainer(
+ completedContainer.getContainerId()));
// 2 launched containers, 1 completed container
- assertEquals(2, schedulerNode1.getNumContainers());
+ assertEquals(2, schedulerNode1.getNumGuaranteedContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getUnallocatedResource());
@@ -389,13 +390,14 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
Resource nmResource =
Resource.newInstance(nm1.getMemory(), nm1.getvCores());
- assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
- assertTrue(
- schedulerNode1.isValidContainer(runningContainer.getContainerId()));
- assertFalse(
- schedulerNode1.isValidContainer(completedContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ amContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ runningContainer.getContainerId()));
+ assertFalse(schedulerNode1.isValidGuaranteedContainer(
+ completedContainer.getContainerId()));
// 2 launched containers, 1 completed container
- assertEquals(2, schedulerNode1.getNumContainers());
+ assertEquals(2, schedulerNode1.getNumGuaranteedContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getUnallocatedResource());
@@ -1700,13 +1702,14 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
Resource nmResource = Resource.newInstance(nm1.getMemory(),
nm1.getvCores());
- assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId()));
- assertTrue(
- schedulerNode1.isValidContainer(runningContainer.getContainerId()));
- assertFalse(
- schedulerNode1.isValidContainer(completedContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ amContainer.getContainerId()));
+ assertTrue(schedulerNode1.isValidGuaranteedContainer(
+ runningContainer.getContainerId()));
+ assertFalse(schedulerNode1.isValidGuaranteedContainer(
+ completedContainer.getContainerId()));
// 2 launched containers, 1 completed container
- assertEquals(2, schedulerNode1.getNumContainers());
+ assertEquals(2, schedulerNode1.getNumGuaranteedContainers());
assertEquals(Resources.subtract(nmResource, usedResources),
schedulerNode1.getUnallocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index fa66cbc..529b298 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -519,7 +519,7 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
totalRes = parseResourceFromString(resSring);
}
}
- when(sn.getTotalResource()).thenReturn(totalRes);
+ when(sn.getCapacity()).thenReturn(totalRes);
when(sn.getUnallocatedResource()).thenReturn(Resources.clone(totalRes));
// TODO, add settings of killable resources when necessary
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
index 964a230..89fb846 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyMockFramework.java
@@ -237,17 +237,17 @@ public class TestProportionalCapacityPreemptionPolicyMockFramework
// Check host resources
Assert.assertEquals(3, this.cs.getAllNodes().size());
SchedulerNode node1 = cs.getSchedulerNode(NodeId.newInstance("n1", 1));
- Assert.assertEquals(100, node1.getTotalResource().getMemorySize());
+ Assert.assertEquals(100, node1.getCapacity().getMemorySize());
Assert.assertEquals(100, node1.getCopiedListOfRunningContainers().size());
Assert.assertNull(node1.getReservedContainer());
SchedulerNode node2 = cs.getSchedulerNode(NodeId.newInstance("n2", 1));
- Assert.assertEquals(0, node2.getTotalResource().getMemorySize());
+ Assert.assertEquals(0, node2.getCapacity().getMemorySize());
Assert.assertEquals(50, node2.getCopiedListOfRunningContainers().size());
Assert.assertNotNull(node2.getReservedContainer());
SchedulerNode node3 = cs.getSchedulerNode(NodeId.newInstance("n3", 1));
- Assert.assertEquals(30, node3.getTotalResource().getMemorySize());
+ Assert.assertEquals(30, node3.getCapacity().getMemorySize());
Assert.assertEquals(100, node3.getCopiedListOfRunningContainers().size());
Assert.assertNull(node3.getReservedContainer());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index ba409b1..0b631a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -289,12 +289,12 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
SchedulerNode mockNode1 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
when(mockNode1.getUnallocatedResource()).thenReturn(emptyResource);
- when(mockNode1.getTotalResource()).thenReturn(fullResource1);
+ when(mockNode1.getCapacity()).thenReturn(fullResource1);
SchedulerNode mockNode2 = mock(SchedulerNode.class);
when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
when(mockNode2.getUnallocatedResource()).thenReturn(emptyResource);
- when(mockNode2.getTotalResource()).thenReturn(fullResource2);
+ when(mockNode2.getCapacity()).thenReturn(fullResource2);
verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
@@ -482,7 +482,7 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
// mock container start
rm1.getRMContext().getScheduler()
- .getSchedulerNode(nm1.getNodeId()).containerStarted(cid);
+ .getSchedulerNode(nm1.getNodeId()).containerLaunched(cid);
// verifies the allocation is made with correct number of tags
Map<String, Long> nodeTags = rm1.getRMContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java
new file mode 100644
index 0000000..b04b277
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerNode.java
@@ -0,0 +1,393 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for SchedulerNode.
+ */
+public class TestSchedulerNode {
+ private final Resource nodeCapacity = Resource.newInstance(1024*10, 4);
+
+ @Test
+ public void testAllocateAndReleaseGuaranteedContainer() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // launch the container on the node
+ schedulerNode.containerLaunched(containerId);
+
+ Assert.assertEquals("The container should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, true);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ }
+
+ @Test
+ public void testAllocateAndReleaseOpportunisticContainer() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.OPPORTUNISTIC, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ nodeCapacity, schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidOpportunisticContainer(containerId));
+
+ // launch the container on the node
+ schedulerNode.containerLaunched(containerId);
+
+ Assert.assertEquals("The container should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, true);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertFalse("The container should have been released",
+ schedulerNode.isValidOpportunisticContainer(containerId));
+ }
+
+ @Test
+ public void testAllocateAndReleaseContainers() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+
+ Resource guaranteedResource = Resource.newInstance(4096, 1);
+ RMContainer guaranteedContainer =
+ createRMContainer(0, guaranteedResource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId guaranteedContainerId = guaranteedContainer.getContainerId();
+
+ // allocate a guaranteed container on the node
+ schedulerNode.allocateContainer(guaranteedContainer);
+
+ Assert.assertEquals("The guaranteed container should have been allocated",
+ guaranteedResource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, guaranteedResource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The guaranteed container should have been allocated" +
+ " but not launched", guaranteedResource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(guaranteedContainerId));
+
+ Resource opportunisticResource = Resource.newInstance(8192, 4);
+ RMContainer opportunisticContainer =
+ createRMContainer(1, opportunisticResource,
+ ExecutionType.OPPORTUNISTIC, schedulerNode.getRMNode());
+ ContainerId opportunisticContainerId =
+ opportunisticContainer.getContainerId();
+
+ // allocate an opportunistic container on the node
+ schedulerNode.allocateContainer(opportunisticContainer);
+
+ Assert.assertEquals("The opportunistic container should have been" +
+ " allocated", opportunisticResource,
+ schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, guaranteedResource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The opportunistic container should also have been" +
+ " allocated but not launched",
+ Resources.add(guaranteedResource, opportunisticResource),
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidOpportunisticContainer(opportunisticContainerId));
+
+ // launch both containers on the node
+ schedulerNode.containerLaunched(guaranteedContainerId);
+ schedulerNode.containerLaunched(opportunisticContainerId);
+
+ Assert.assertEquals("Both containers should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release both containers
+ schedulerNode.releaseContainer(guaranteedContainerId, true);
+ schedulerNode.releaseContainer(opportunisticContainerId, true);
+
+ Assert.assertEquals("The guaranteed container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The opportunistic container should have been released",
+ 0, schedulerNode.getNumOpportunisticContainers());
+ Assert.assertEquals("The guaranteed container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ Assert.assertEquals("The opportunistic container should have been released",
+ Resources.none(), schedulerNode.getOpportunisticResourceAllocated());
+ Assert.assertFalse("The guaranteed container should have been released",
+ schedulerNode.isValidGuaranteedContainer(guaranteedContainerId));
+ Assert.assertFalse("The opportunistic container should have been released",
+ schedulerNode.isValidOpportunisticContainer(opportunisticContainerId));
+ }
+
+ @Test
+ public void testReleaseLaunchedContainerNotAsNode() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // launch the container on the node
+ schedulerNode.containerLaunched(containerId);
+
+ Assert.assertEquals("The container should have been launched",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, false);
+ Assert.assertEquals("The container should not have been released",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should not have been released",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertTrue("The container should not have been released",
+ schedulerNode.isValidGuaranteedContainer(containerId));
+ }
+
+ @Test
+ public void testReleaseUnlaunchedContainerAsNode() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched",
+ resource, schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // make sure the container is not launched yet
+ Assert.assertEquals("The container should not be launched already",
+ resource, schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, true);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ Assert.assertFalse("The container should have been released",
+ schedulerNode.isValidGuaranteedContainer(containerId));
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+ }
+
+ @Test
+ public void testReleaseUnlaunchedContainerNotAsNode() {
+ SchedulerNode schedulerNode = createSchedulerNode(nodeCapacity);
+ Resource resource = Resource.newInstance(4096, 1);
+ RMContainer container = createRMContainer(0, resource,
+ ExecutionType.GUARANTEED, schedulerNode.getRMNode());
+ ContainerId containerId = container.getContainerId();
+
+ // allocate a container on the node
+ schedulerNode.allocateContainer(container);
+
+ Assert.assertEquals("The container should have been allocated",
+ resource, schedulerNode.getAllocatedResource());
+ Assert.assertEquals("Incorrect remaining resource accounted.",
+ Resources.subtract(nodeCapacity, resource),
+ schedulerNode.getUnallocatedResource());
+ Assert.assertEquals("The container should have been allocated" +
+ " but not launched", resource,
+ schedulerNode.getResourceAllocatedPendingLaunch());
+ Assert.assertEquals("The container should have been allocated",
+ 1, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertTrue(
+ schedulerNode.isValidGuaranteedContainer(containerId));
+
+ // make sure the container is not launched yet
+ Assert.assertEquals("The container should not have been launched",
+ resource, schedulerNode.getResourceAllocatedPendingLaunch());
+
+ // release the container
+ schedulerNode.releaseContainer(containerId, false);
+ Assert.assertEquals("The container should have been released",
+ 0, schedulerNode.getNumGuaranteedContainers());
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getAllocatedResource());
+ Assert.assertFalse("The container should have been released",
+ schedulerNode.isValidGuaranteedContainer(containerId));
+ Assert.assertEquals("The container should have been released",
+ Resources.none(), schedulerNode.getResourceAllocatedPendingLaunch());
+ }
+
+ private SchedulerNode createSchedulerNode(Resource capacity) {
+ NodeId nodeId = NodeId.newInstance("localhost", 0);
+
+ RMNode rmNode = mock(RMNode.class);
+ when(rmNode.getNodeID()).thenReturn(nodeId);
+ when(rmNode.getHostName()).thenReturn(nodeId.getHost());
+ when(rmNode.getTotalCapability()).thenReturn(capacity);
+ when(rmNode.getRackName()).thenReturn("/default");
+ when(rmNode.getHttpAddress()).thenReturn(nodeId.getHost());
+ when(rmNode.getNodeAddress()).thenReturn(nodeId.getHost());
+
+ return new SchedulerNodeForTest(rmNode);
+ }
+
+ private static RMContainerImpl createRMContainer(long containerId,
+ Resource resource, ExecutionType executionType, RMNode node) {
+ Container container =
+ createContainer(containerId, resource, executionType, node);
+
+ Dispatcher dispatcher = new AsyncDispatcher();
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(dispatcher);
+ when(rmContext.getSystemMetricsPublisher()).
+ thenReturn(new NoOpSystemMetricPublisher());
+ when(rmContext.getYarnConfiguration()).
+ thenReturn(new YarnConfiguration());
+ when(rmContext.getContainerAllocationExpirer()).
+ thenReturn(new ContainerAllocationExpirer(dispatcher));
+ when(rmContext.getRMApplicationHistoryWriter()).
+ thenReturn(new RMApplicationHistoryWriter());
+
+ return new RMContainerImpl(container, null,
+ container.getId().getApplicationAttemptId(),
+ node.getNodeID(), "test", rmContext);
+ }
+
+ private static Container createContainer(long containerId, Resource resource,
+ ExecutionType executionType, RMNode node) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.
+ newInstance(ApplicationId.newInstance(0, 0), 0);
+ ContainerId cId =
+ ContainerId.newContainerId(appAttemptId, containerId);
+ Container container = Container.newInstance(
+ cId, node.getNodeID(), node.getNodeAddress(), resource,
+ Priority.newInstance(0), null, executionType);
+ return container;
+ }
+
+
+ /**
+ * A test implementation of SchedulerNode for the purpose of testing
+ * SchedulerNode only. Resource reservation is scheduler-dependent,
+ * and therefore not covered here.
+ */
+ private static final class SchedulerNodeForTest extends SchedulerNode {
+ SchedulerNodeForTest(RMNode node) {
+ super(node, false);
+ }
+
+ @Override
+ public void reserveResource(SchedulerApplicationAttempt attempt,
+ SchedulerRequestKey schedulerKey, RMContainer container) {
+ }
+
+ @Override
+ public void unreserveResource(SchedulerApplicationAttempt attempt) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index e77d8e2..9e15856 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4131,7 +4131,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
// Check total resource of scheduler node is also changed to 1 GB 1 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals("Total Resource Memory Size should be 1GB", 1 * GB,
totalResource.getMemorySize());
Assert.assertEquals("Total Resource Virtual Cores should be 1", 1,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 840d30d..9aeb946 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -229,8 +229,8 @@ public class TestCapacitySchedulerAsyncScheduling {
// nm1 runs 1 container(app1-container_01/AM)
// nm2 runs 1 container(app1-container_02)
- Assert.assertEquals(1, sn1.getNumContainers());
- Assert.assertEquals(1, sn2.getNumContainers());
+ Assert.assertEquals(1, sn1.getNumGuaranteedContainers());
+ Assert.assertEquals(1, sn2.getNumGuaranteedContainers());
// kill app attempt1
scheduler.handle(
@@ -325,8 +325,8 @@ public class TestCapacitySchedulerAsyncScheduling {
// nm1 runs 3 containers(app1-container_01/AM, app1-container_02,
// app2-container_01/AM)
// nm2 runs 1 container(app1-container_03)
- Assert.assertEquals(3, sn1.getNumContainers());
- Assert.assertEquals(1, sn2.getNumContainers());
+ Assert.assertEquals(3, sn1.getNumGuaranteedContainers());
+ Assert.assertEquals(1, sn2.getNumGuaranteedContainers());
// reserve 1 container(app1-container_04) for app1 on nm1
ResourceRequest rr2 = ResourceRequest
@@ -639,7 +639,7 @@ public class TestCapacitySchedulerAsyncScheduling {
// nm1 runs 2 container(container_01/AM, container_02)
allocateAndLaunchContainers(am, nm1, rm, 1,
Resources.createResource(6 * GB), 0, 2);
- Assert.assertEquals(2, sn1.getNumContainers());
+ Assert.assertEquals(2, sn1.getNumGuaranteedContainers());
Assert.assertEquals(1 * GB, sn1.getUnallocatedResource().getMemorySize());
// app asks 5 * 2G container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 04bb791..6215ce8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -385,7 +385,7 @@ public class TestLeafQueue {
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
assertEquals(
- (int)(node_0.getTotalResource().getMemorySize() * a.getCapacity()) - (1*GB),
+ (int)(node_0.getCapacity().getMemorySize() * a.getCapacity()) - (1*GB),
a.getMetrics().getAvailableMB());
}
@@ -684,7 +684,7 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(0*GB, a.getMetrics().getAllocatedMB());
- assertEquals((int)(a.getCapacity() * node_0.getTotalResource().getMemorySize()),
+ assertEquals((int)(a.getCapacity() * node_0.getCapacity().getMemorySize()),
a.getMetrics().getAvailableMB());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-6672. Add NM preemption of
opportunistic containers when utilization goes high.
Posted by ha...@apache.org.
YARN-6672. Add NM preemption of opportunistic containers when utilization goes high.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/124b378f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/124b378f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/124b378f
Branch: refs/heads/YARN-1011
Commit: 124b378ff2d54db4d6dcac032a816b6cf26d7209
Parents: 23ad2fb
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jul 2 15:37:47 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:48:55 2018 -0700
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 85 ++-
.../AllocationBasedResourceTracker.java | 5 +
.../scheduler/ContainerScheduler.java | 59 +-
.../scheduler/ContainerSchedulerEventType.java | 4 +-
...rSchedulerOverallocationPreemptionEvent.java | 45 ++
.../scheduler/NMAllocationPreemptionPolicy.java | 54 ++
.../scheduler/ResourceUtilizationTracker.java | 7 +
...shotBasedOverAllocationPreemptionPolicy.java | 81 +++
.../BaseContainerManagerTest.java | 126 ++++
.../TestContainerSchedulerQueuing.java | 167 +++--
...estContainerSchedulerWithOverAllocation.java | 650 +++++++++++--------
...shotBasedOverAllocationPreemptionPolicy.java | 259 ++++++++
12 files changed, 1229 insertions(+), 313 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 8f72b56..c36dfd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -20,15 +20,20 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerOverallocationPreemptionEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPreemptionPolicy;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPreemptionPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,6 +126,7 @@ public class ContainersMonitorImpl extends AbstractService implements
private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
+ private NMAllocationPreemptionPolicy overAllocationPreemptionPolicy;
private volatile boolean stopped = false;
@@ -375,6 +381,9 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPolicy =
createOverAllocationPolicy(resourceThresholds);
+ this.overAllocationPreemptionPolicy = createOverAllocationPreemptionPolicy(
+ overAllocationPreemptionThresholds, overAlloctionPreemptionCpuCount);
+
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -387,6 +396,12 @@ public class ContainersMonitorImpl extends AbstractService implements
return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
}
+ private NMAllocationPreemptionPolicy createOverAllocationPreemptionPolicy(
+ ResourceThresholds resourceThresholds, int maxTimesCpuOverLimit) {
+ return new SnapshotBasedOverAllocationPreemptionPolicy(
+ resourceThresholds, maxTimesCpuOverLimit, this);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -673,9 +688,7 @@ public class ContainersMonitorImpl extends AbstractService implements
setLatestContainersUtilization(trackedContainersUtilization);
// check opportunity to start containers if over-allocation is on
- if (context.isOverAllocationEnabled()) {
- attemptToStartContainersUponLowUtilization();
- }
+ checkUtilization();
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1076,13 +1089,46 @@ public class ContainersMonitorImpl extends AbstractService implements
return overAllocationPolicy;
}
+ public NMAllocationPreemptionPolicy getOverAllocationPreemptionPolicy() {
+ return overAllocationPreemptionPolicy;
+ }
+
private void setLatestContainersUtilization(ResourceUtilization utilization) {
this.latestContainersUtilization = new ContainersResourceUtilization(
- utilization, System.currentTimeMillis());
+ utilization, Time.now());
}
+ /**
+ * Check the resource utilization of the node. If the utilization is below
+ * the over-allocation threshold, {@link ContainerScheduler} is notified to
+ * launch OPPORTUNISTIC containers that are being queued to bring the
+ * utilization up to the over-allocation threshold. If the utilization
+ * is above the preemption threshold, {@link ContainerScheduler} is notified
+ * to preempt running OPPORTUNISTIC containers in order to bring the node
+ * utilization down to the preemption threshold.
+ * @return true if the utilization is below the over-allocation threshold or
+ * above the preemption threshold
+ * false otherwise
+ */
@VisibleForTesting
- public void attemptToStartContainersUponLowUtilization() {
+ public boolean checkUtilization() {
+ if (context.isOverAllocationEnabled()) {
+ return checkLowUtilization() || checkHighUtilization();
+ }
+ return false;
+ }
+
+ /**
+ * Check if the node resource utilization is below the over-allocation
+ * threshold. If so, a {@link ContainerSchedulerEvent} is
+ * generated so that OPPORTUNISTIC containers that are being queued can
+ * be launched by {@link ContainerScheduler} with over-allocation and
+ * the node utilization can be brought up to the over-allocation threshold
+ * @return true if the node utilization is below the over-allocation threshold
+ * false otherwise
+ */
+ private boolean checkLowUtilization() {
+ boolean opportunisticContainersToStart = false;
if (getContainerOverAllocationPolicy() != null) {
Resource available = getContainerOverAllocationPolicy()
.getAvailableResources();
@@ -1091,8 +1137,37 @@ public class ContainersMonitorImpl extends AbstractService implements
eventDispatcher.getEventHandler().handle(
new ContainerSchedulerEvent(null,
ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
+ opportunisticContainersToStart = true;
+ LOG.info("Node utilization is below its over-allocation threshold. " +
+ "Inform container scheduler to launch opportunistic containers.");
}
}
+ return opportunisticContainersToStart;
+ }
+
+ /**
+ * Check if the node resource utilization is over the preemption threshold.
+ * If so, a {@link ContainerSchedulerOverallocationPreemptionEvent} is
+ * generated so that OPPORTUNISTIC containers can be preempted by
+ * {@link ContainerScheduler} to reclaim resources in order to bring the
+ * node utilization down to the preemption threshold.
+ * @return true if the node utilization is over the preemption threshold
+ * false otherwise
+ */
+ private boolean checkHighUtilization() {
+ ResourceUtilization overLimit = getOverAllocationPreemptionPolicy()
+ .getResourcesToReclaim();
+
+ boolean opportunisticContainersToPreempt = false;
+
+ if (overLimit.getPhysicalMemory() > 0 || overLimit.getCPU() > 0) {
+ opportunisticContainersToPreempt = true;
+ eventDispatcher.getEventHandler().handle(
+ new ContainerSchedulerOverallocationPreemptionEvent(overLimit));
+ LOG.info("Node utilization is over the preemption threshold. " +
+ "Inform container scheduler to reclaim {}", overLimit);
+ }
+ return opportunisticContainersToPreempt;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
index a3e19eb..7840fef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -53,6 +53,11 @@ public class AllocationBasedResourceTracker
*/
@Override
public ResourceUtilization getCurrentUtilization() {
+ return getTotalAllocation();
+ }
+
+ @Override
+ public ResourceUtilization getTotalAllocation() {
return this.containersAllocation;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0d7de67..846efc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -222,12 +222,60 @@ public class ContainerScheduler extends AbstractService implements
// node utilization is low.
startOpportunisticContainers(utilizationTracker.getAvailableResources());
break;
+ case PREEMPT_CONTAINERS:
+ if (event instanceof ContainerSchedulerOverallocationPreemptionEvent) {
+ preemptOpportunisticContainers(
+ (ContainerSchedulerOverallocationPreemptionEvent) event);
+ } else {
+ LOG.error(
+ "Unknown event type on Preempt containers: {}", event.getType());
+ }
+ break;
default:
- LOG.error("Unknown event arrived at ContainerScheduler: "
- + event.toString());
+ LOG.error("Unknown event arrived at ContainerScheduler: {}", event);
}
}
+ private void preemptOpportunisticContainers(
+ ContainerSchedulerOverallocationPreemptionEvent event) {
+ ResourceUtilization resourcesToReclaim =
+ getResourcesToReclaim(event.getResourcesOverPreemptionThresholds());
+
+ List<Container> oppContainersToReclaim =
+ pickOpportunisticContainersToReclaimResources(
+ resourcesToReclaim);
+
+ killOpportunisticContainers(oppContainersToReclaim);
+ }
+
+ /**
+ * Get the amount of resources that need to be reclaimed by preempting
+ * OPPORTUNISTIC containers considering the amount of resources that
+ * are over the preemption thresholds and over the capacity of the node.
+ * When the node is not being over-allocated, its resource utilization
+ * can safely go to 100% without any OPPORTUNISTIC containers being killed.
+ */
+ private ResourceUtilization getResourcesToReclaim(
+ ResourceUtilization resourcesOverPreemptionThresholds) {
+ ResourceUtilization totalAllocation = ResourceUtilization.newInstance(
+ utilizationTracker.getTotalAllocation());
+ getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
+ totalAllocation);
+ ResourceUtilization overAllocatedResources =
+ ResourceUtilization.newInstance(
+ Math.max(0, totalAllocation.getPhysicalMemory()),
+ Math.max(0, totalAllocation.getVirtualMemory()),
+ Math.max(0, totalAllocation.getCPU()));
+
+ return ResourceUtilization.newInstance(
+ Math.min(overAllocatedResources.getPhysicalMemory(),
+ resourcesOverPreemptionThresholds.getPhysicalMemory()),
+ Math.min(overAllocatedResources.getVirtualMemory(),
+ resourcesOverPreemptionThresholds.getVirtualMemory()),
+ Math.min(overAllocatedResources.getCPU(),
+ resourcesOverPreemptionThresholds.getCPU()));
+ }
+
/**
* We assume that the ContainerManager has already figured out what kind
* of update this is.
@@ -601,8 +649,9 @@ public class ContainerScheduler extends AbstractService implements
@SuppressWarnings("unchecked")
private void reclaimOpportunisticContainerResources() {
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources();
+ pickOpportunisticContainersToReclaimResources(resourcesToFreeUp);
killOpportunisticContainers(extraOppContainersToReclaim);
}
@@ -642,12 +691,12 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources() {
+ private List<Container> pickOpportunisticContainersToReclaimResources(
+ ResourceUtilization resourcesToFreeUp) {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 9ad4f91..f76727d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -30,5 +30,7 @@ public enum ContainerSchedulerEventType {
CONTAINER_PAUSED,
RECOVERY_COMPLETED,
// Producer: Containers Monitor when over-allocation is on
- SCHEDULE_CONTAINERS
+ SCHEDULE_CONTAINERS,
+ // Producer: Containers Monitor when over-allocation is on
+ PREEMPT_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java
new file mode 100644
index 0000000..547036a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+
+/**
+ * A {@link ContainerSchedulerEvent} generated by {@link ContainersMonitorImpl}
+ * when overallocation is turned on and the utilization goes over the
+ * proactive preemption thresholds.
+ */
+public class ContainerSchedulerOverallocationPreemptionEvent
+ extends ContainerSchedulerEvent {
+ private final ResourceUtilization resourcesOverPreemptionThresholds;
+ /**
+ * Create instance of Event.
+ *
+ * @param toFree resource to free up.
+ */
+ public ContainerSchedulerOverallocationPreemptionEvent(
+ ResourceUtilization toFree) {
+ super(null, ContainerSchedulerEventType.PREEMPT_CONTAINERS);
+ this.resourcesOverPreemptionThresholds = toFree;
+ }
+
+ public ResourceUtilization getResourcesOverPreemptionThresholds() {
+ return resourcesOverPreemptionThresholds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java
new file mode 100644
index 0000000..61f6298
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * Keeps track of containers utilization over time and determines how many
+ * resources need to be reclaimed by preempting opportunistic containers
+ * when over-allocation is turned on.
+ */
+public abstract class NMAllocationPreemptionPolicy {
+ private final ResourceThresholds overAllocationPreemptionThresholds;
+ private final ContainersMonitor containersMonitor;
+
+ public NMAllocationPreemptionPolicy(
+ ResourceThresholds preemptionThresholds,
+ ContainersMonitor containersMonitor) {
+ this.containersMonitor = containersMonitor;
+ this.overAllocationPreemptionThresholds = preemptionThresholds;
+ }
+
+ /**
+ * Get the amount of resources to reclaim by preempting opportunistic
+ * containers when over-allocation is turned on.
+ * @return the amount of resources to be reclaimed
+ */
+ public abstract ResourceUtilization getResourcesToReclaim();
+
+ public ContainersMonitor getContainersMonitor() {
+ return containersMonitor;
+ }
+
+ public ResourceThresholds getOverAllocationPreemptionThresholds() {
+ return overAllocationPreemptionThresholds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 7a7c78e..9be7574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -39,6 +39,13 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
+ * Get the total amount of resources allocated to running containers
+ * in terms of resource utilization.
+ * @return ResourceUtilization resource allocation
+ */
+ ResourceUtilization getTotalAllocation();
+
+ /**
* Get the amount of resources currently available to launch containers.
* @return Resource resources available to launch containers
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java
new file mode 100644
index 0000000..188a108
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * An implementation of {@link NMAllocationPreemptionPolicy} based on the
+ * snapshot of the latest containers utilization to determine how many
+ * resources need to be reclaimed by preempting opportunistic containers
+ * when over-allocation is turned on.
+ */
+public class SnapshotBasedOverAllocationPreemptionPolicy
+ extends NMAllocationPreemptionPolicy {
+ private final int absoluteMemoryPreemptionThresholdMb;
+ private final float cpuPreemptionThreshold;
+ private final int maxTimesCpuOverPreemption;
+ private int timesCpuOverPreemption;
+
+ public SnapshotBasedOverAllocationPreemptionPolicy(
+ ResourceThresholds preemptionThresholds,
+ int timesCpuOverPreemptionThreshold,
+ ContainersMonitor containersMonitor) {
+ super(preemptionThresholds, containersMonitor);
+ int memoryCapacityMb = (int)
+ (containersMonitor.getPmemAllocatedForContainers() / (1024 * 1024));
+ this.absoluteMemoryPreemptionThresholdMb = (int)
+ (preemptionThresholds.getMemoryThreshold() * memoryCapacityMb);
+ this.cpuPreemptionThreshold = preemptionThresholds.getCpuThreshold();
+ this.maxTimesCpuOverPreemption = timesCpuOverPreemptionThreshold;
+ }
+
+ @Override
+ public ResourceUtilization getResourcesToReclaim() {
+ ResourceUtilization utilization =
+ getContainersMonitor().getContainersUtilization(true).getUtilization();
+
+ int memoryOverLimit = utilization.getPhysicalMemory() -
+ absoluteMemoryPreemptionThresholdMb;
+ float vcoreOverLimit = utilization.getCPU() - cpuPreemptionThreshold;
+
+ if (vcoreOverLimit > 0) {
+ timesCpuOverPreemption++;
+ if (timesCpuOverPreemption > maxTimesCpuOverPreemption) {
+ timesCpuOverPreemption = 0;
+ } else {
+ // report no over limit for cpu if # of times CPU is over the preemption
+ // threshold is not greater the max number of times allowed
+ vcoreOverLimit = 0;
+ }
+ } else {
+ // reset the counter when cpu utilization goes under the preemption
+ // threshold before the max times allowed is reached
+ timesCpuOverPreemption = 0;
+ }
+
+ // sanitize so that zero is returned if the utilization is below
+ // the preemption threshold
+ vcoreOverLimit = Math.max(0, vcoreOverLimit);
+ memoryOverLimit = Math.max(0, memoryOverLimit);
+
+ return ResourceUtilization.newInstance(memoryOverLimit, 0, vcoreOverLimit);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 92613ed..9f1506a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,7 +26,12 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -559,4 +564,125 @@ public abstract class BaseContainerManagerTest {
ContainerId.newContainerId(appAttemptId, cId);
return containerId;
}
+
+ /**
+ * A test implementation of {@link ContainersMonitor} that allows control of
+ * the current resource utilization.
+ */
+ protected static class ContainerMonitorForTest
+ extends ContainersMonitorImpl {
+ private static final int NM_CONTAINERS_VCORES = 4;
+ private static final int NM_CONTAINERS_MEMORY_MB = 2048;
+
+ private ResourceUtilization containerResourceUsage =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
+ ContainerMonitorForTest(ContainerExecutor exec,
+ AsyncDispatcher dispatcher, Context context) {
+ super(exec, dispatcher, context);
+ }
+
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
+ }
+
+ @Override
+ public long getVmemAllocatedForContainers() {
+ float pmemRatio = getConfig().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ return (long) (pmemRatio * getPmemAllocatedForContainers());
+ }
+
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return NM_CONTAINERS_VCORES;
+ }
+
+ @Override
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ return new ContainersMonitor.ContainersResourceUtilization(
+ containerResourceUsage, Time.now());
+ }
+
+ @Override
+ protected void checkOverAllocationPrerequisites() {
+ // do not check
+ }
+
+ public void setContainerResourceUsage(
+ ResourceUtilization containerResourceUsage) {
+ this.containerResourceUsage = containerResourceUsage;
+ }
+ }
+
+ /**
+ * A test implementation of {@link ContainerManager} that allows its
+ * internal event queue to be drained for synchronization purpose,
+ * and an out-of-band check of the node resource utilization so that
+ * the node utilization can stay between the over-allocation threshold
+ * and the preemption threshold.
+ */
+ protected static class ContainerManagerForTest
+ extends ContainerManagerImpl {
+
+ private final String user;
+
+ public ContainerManagerForTest(
+ Context context, ContainerExecutor exec,
+ DeletionService deletionContext,
+ NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler, String user) {
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler);
+ this.user = user;
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ @Override
+ protected AsyncDispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(
+ ContainerExecutor exec) {
+ return new ContainerMonitorForTest(exec, dispatcher, context);
+ }
+
+ /**
+ * Check the node resource utilization out-of-band. If the utilization is
+ * below the over-allocation threshold, queued OPPORTUNISTIC containers
+ * will be launched to bring the node utilization up to the over-allocation
+ * threshold. If the utilization is above the preemption threshold, running
+ * OPPORTUNISTIC containers will be killed to bring the utilization down to
+ * the preemption threshold.
+ */
+ public void checkNodeResourceUtilization() {
+ ((ContainerMonitorForTest) getContainersMonitor()).checkUtilization();
+ drainAsyncEvents();
+ }
+
+ /**
+ * Drain the internal event queue.
+ */
+ public void drainAsyncEvents() {
+ ((DrainDispatcher) dispatcher).await();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/124b378f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 70066c6..d5c09b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ConcurrentMap;
import com.google.common.base.Supplier;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse;
@@ -38,18 +37,15 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
@@ -65,8 +61,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
@@ -131,47 +125,8 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
@Override
protected ContainerManagerImpl createContainerManager(
DeletionService delSrvc) {
- return new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics, dirsHandler) {
-
- @Override
- protected UserGroupInformation getRemoteUgi() throws YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
- .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
- .getKeyId()));
- return ugi;
- }
-
- @Override
- protected ContainersMonitor createContainersMonitor(
- ContainerExecutor exec) {
- return new ContainersMonitorImpl(exec, dispatcher, this.context) {
- // Define resources available for containers to be executed.
- @Override
- public long getPmemAllocatedForContainers() {
- return 2048 * 1024 * 1024L;
- }
-
- @Override
- public long getVmemAllocatedForContainers() {
- float pmemRatio = getConfig().getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- return (long) (pmemRatio * getPmemAllocatedForContainers());
- }
-
- @Override
- public long getVCoresAllocatedForContainers() {
- return 4;
- }
- };
- }
- };
+ return new ContainerManagerForTest(context, exec, delSrvc,
+ nodeStatusUpdater, metrics, dirsHandler, user);
}
@Override
@@ -393,6 +348,37 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
containerScheduler.getNumQueuedGuaranteedContainers());
Assert.assertEquals(2,
containerScheduler.getNumQueuedOpportunisticContainers());
+
+ // we have just one container that requested 2048 MB of memory and 1 vcore
+ // running, its resource utilization is zero.
+ // check the node resource utilization and the two OPPORTUNISTIC containers
+ // that are being queued should stay in the queue because over-allocation
+ // is turn off.
+ ((ContainerMonitorForTest) containerManager.getContainersMonitor())
+ .setContainerResourceUsage(
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ ContainerId containerId0 = createContainerId(0);
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(containerId0)) {
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
+ } else {
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
+ }
+ }
+ containerScheduler = containerManager.getContainerScheduler();
+ // Ensure two containers are properly queued.
+ Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
+ Assert.assertEquals(0,
+ containerScheduler.getNumQueuedGuaranteedContainers());
+ Assert.assertEquals(2,
+ containerScheduler.getNumQueuedOpportunisticContainers());
}
/**
@@ -476,6 +462,91 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
/**
+ * Start two OPPORTUNISTIC containers which together ask for all
+ * the allocations available on the node. When the node resource
+ * utilization goes over the preemption thresholds, neither of
+ * the containers should be preempted because there is no
+ * over-allocation at the moment and they can safely use up all
+ * the resources availabe on the node.
+ */
+ @Test
+ public void testNoOpportunisticContainerPreemptionUponHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ // start two OPPORTUNISTIC containers that together takes up
+ // all allocations of the node. They two can be launched immediately
+ // because there is enough free allocation. When they uses up
+ // all their resource allocations, that is, the node is fully
+ // utilized, none of the OPPORTUNISTIC containers shall be killed
+ // because the node is not being over-allocated
+ List<StartContainerRequest> list = new ArrayList<>();
+ list.add(StartContainerRequest.newInstance(
+ recordFactory.newRecordInstance(ContainerLaunchContext.class),
+ createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(1024, 2),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.OPPORTUNISTIC)));
+ list.add(StartContainerRequest.newInstance(
+ recordFactory.newRecordInstance(ContainerLaunchContext.class),
+ createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(1024, 2),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.OPPORTUNISTIC)));
+
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+
+ // the two OPPORTUNISTIC containers shall be launched immediately
+ // because there is just enough allocation to launch them both.
+ BaseContainerManagerTest.waitForContainerState(containerManager,
+ createContainerId(0),
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
+ BaseContainerManagerTest.waitForContainerState(containerManager,
+ createContainerId(1),
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
+
+ // Ensure all containers are running.
+ List<ContainerId> statList = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ statList.add(createContainerId(i));
+ }
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+ }
+
+ // we have two containers running, both of which are using all of
+ // their allocations. The node is being fully utilized in terms
+ // of both memory and CPU
+ ((ContainerMonitorForTest) containerManager.getContainersMonitor())
+ .setContainerResourceUsage(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // the two running OPPORTUNISTIC containers shall continue to run
+ // because when the node is not be over-allocated, it is safe to
+ // let the containers use up all the resources, no OPPORTUNISTIC
+ // containers shall be preempted
+ containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+ }
+ }
+
+ /**
* Submit two OPPORTUNISTIC and one GUARANTEED containers. The resources
* requests by each container as such that only one can run in parallel.
* Thus, the OPPORTUNISTIC container that started running, will be
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-6675. Add NM support to
launch opportunistic containers based on overallocation. Contributed by Haibo
Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index 6b3ac67..f24df38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -66,8 +67,8 @@ public class TestContainerSchedulerRecovery {
@Mock private ContainerId containerId;
- @Mock private AllocationBasedResourceUtilizationTracker
- allocationBasedResourceUtilizationTracker;
+ @Mock private AllocationBasedResourceTracker
+ allocationBasedResourceTracker;
@InjectMocks private ContainerScheduler tempContainerScheduler =
new ContainerScheduler(context, dispatcher, metrics, 0);
@@ -85,12 +86,13 @@ public class TestContainerSchedulerRecovery {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
when(container.getContainerId()).thenReturn(containerId);
+ when(container.getResource()).thenReturn(Resource.newInstance(1024, 1));
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(containerId.getApplicationAttemptId().getApplicationId())
.thenReturn(appId);
when(containerId.getContainerId()).thenReturn(123L);
- doNothing().when(allocationBasedResourceUtilizationTracker)
- .addContainerResources(container);
+ doNothing().when(allocationBasedResourceTracker)
+ .containerLaunched(container);
}
@After public void tearDown() {
@@ -112,8 +114,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
@@ -132,8 +134,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, GUARANTEED,
@@ -152,8 +154,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
@@ -172,8 +174,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, GUARANTEED,
@@ -192,8 +194,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
@@ -212,8 +214,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, GUARANTEED,
@@ -232,8 +234,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
@@ -252,8 +254,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, GUARANTEED,
@@ -272,8 +274,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
@@ -292,8 +294,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as GUARANTEED but no executionType set,
@@ -311,8 +313,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED but no executionType set,
@@ -330,7 +332,7 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08116cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
new file mode 100644
index 0000000..384b116
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -0,0 +1,1121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+/**
+ * Test ContainerScheduler behaviors when NM overallocation is turned on.
+ */
+public class TestContainerSchedulerWithOverAllocation
+ extends BaseContainerManagerTest {
+ private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
+ private static final int NM_CONTAINERS_VCORES = 4;
+ private static final int NM_CONTAINERS_MEMORY_MB = 2048;
+
+ static {
+ LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
+ }
+
+ public TestContainerSchedulerWithOverAllocation()
+ throws UnsupportedFileSystemException {
+ }
+
+ @Override
+ protected ContainerExecutor createContainerExecutor() {
+ DefaultContainerExecutor exec =
+ new LongRunningContainerSimulatingContainerExecutor();
+ exec.setConf(conf);
+ return exec;
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(
+ DeletionService delSrvc) {
+ return new LongRunningContainerSimulatingContainersManager(
+ context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler, user);
+ }
+
+ @Override
+ public void setup() throws IOException {
+ conf.setInt(
+ YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
+ NM_OPPORTUNISTIC_QUEUE_LIMIT);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
+ 0.75f);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
+ 0.75f);
+ super.setup();
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC container, which in aggregate do
+ * not exceed the capacity of the node. Both containers are expected to start
+ * running immediately.
+ */
+ @Test
+ public void testStartMultipleContainersWithoutOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ StartContainersRequest allRequests = StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() { {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ } }
+ );
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers whose utilization
+ * is very low relative to their resource request, resulting in a low node
+ * utilization. Then start another OPPORTUNISTIC containers which requests
+ * more than what's left unallocated on the node. Due to overallocation
+ * being turned on and node utilization being low, the second OPPORTUNISTIC
+ * container is also expected to be launched immediately.
+ */
+ @Test
+ public void testStartOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started immediately because there
+ // are (memory: 1024, vcore: 0.625) available based on over-allocation
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers which utilizes most
+ * of the resources they requested, resulting in a high node utilization.
+ * Then start another OPPORTUNISTIC containers which requests more than what's
+ * left unallocated on the node. Because of the high resource utilization on
+ * the node, the projected utilization, if we were to start the second
+ * OPPORTUNISTIC container immediately, will go over the NM overallocation
+ * threshold, so the second OPPORTUNISTIC container is expected to be queued.
+ */
+ @Test
+ public void testQueueOppContainerWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1500, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+ // this container will not start immediately because there is not
+ // enough resource available at the moment either in terms of
+ // resources unallocated or in terms of the actual utilization
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+ }
+
+ /**
+ * Start two GUARANTEED containers which in aggregate takes up the whole node
+ * capacity, yet whose utilization is low relative to their resource request,
+ * resulting in a low node resource utilization. Then try to start another
+ * OPPORTUNISTIC containers. Because the resource utilization across the node
+ * is low and overallocation being turned on, the OPPORTUNISTIC container is
+ * expected to be launched immediately even though there is no resources left
+ * unallocated.
+ */
+ @Test
+ public void testStartOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(800, 0, 1.0f/8));
+
+ // start a container when there is no resources left unallocated.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started because there is resources
+ // available because the actual utilization is very low
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ /**
+ * Start two GUARANTEED containers which in aggregate take up the whole node
+ * capacity and fully utilize the resources they requested. Then try to start
+ * four OPPORTUNISTIC containers of which three will be queued and one will be
+ * killed because of the max queue length is 3.
+ */
+ @Test
+ public void testQueueOppContainersWithFullUtilization() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers are fully utilizing their resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f/8));
+
+ // start more OPPORTUNISTIC containers than what the OPPORTUNISTIC container
+ // queue can hold when there is no unallocated resource left.
+ List<StartContainerRequest> moreContainerRequests =
+ new ArrayList<>(NM_OPPORTUNISTIC_QUEUE_LIMIT + 1);
+ for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
+ moreContainerRequests.add(
+ createStartContainerRequest(2 + a,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ containerManager.startContainers(
+ StartContainersRequest.newInstance(moreContainerRequests));
+
+ // All OPPORTUNISTIC containers but the last one should be queued.
+ // The last OPPORTUNISTIC container to launch should be killed.
+ BaseContainerManagerTest.waitForContainerState(
+ containerManager, createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT + 2),
+ ContainerState.COMPLETE);
+
+ HashMap<ContainerId, ContainerSubState> expectedContainerStatus =
+ new HashMap<>();
+ expectedContainerStatus.put(
+ createContainerId(0), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(1), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT),
+ ContainerSubState.DONE);
+ for (int i = 0; i < NM_OPPORTUNISTIC_QUEUE_LIMIT; i++) {
+ expectedContainerStatus.put(
+ createContainerId(i + 2), ContainerSubState.SCHEDULED);
+ }
+ verifyContainerStatuses(expectedContainerStatus);
+ }
+
+ /**
+ * Start two GUARANTEED containers that together does not take up the
+ * whole node. Then try to start one OPPORTUNISTIC container that will
+ * fit into the remaining unallocated space on the node.
+ * The OPPORTUNISTIC container is expected to start even though the
+ * current node utilization is above the NM overallocation threshold,
+ * because it's always safe to launch containers as long as the node
+ * has not been fully allocated.
+ */
+ @Test
+ public void testStartOppContainerWithHighUtilizationNoOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1200, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(400, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is above the over-allocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1600, 0, 1.0f/2));
+
+ // start a container that can just fit in the remaining unallocated space
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(400, 1), false))
+ ));
+
+ // the OPPORTUNISTIC container can be safely launched even though
+ // the container utilization is above the NM overallocation threshold
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers first whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization. Then
+ * try to start a GUARANTEED container which requests more than what's left
+ * unallocated on the node. Because the node utilization is low and NM
+ * overallocation is turned on, the GUARANTEED container is expected to be
+ * started immediately without killing any running OPPORTUNISTIC containers.
+ */
+ @Test
+ public void testKillNoOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers.
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers whose utilization will be high relative
+ * to the resources they requested, resulting in a high node utilization.
+ * Then try to start a GUARANTEED container which requests more than what's
+ * left unallocated on the node. Because the node is under high utilization,
+ * the second OPPORTUNISTIC container is expected to be killed in order to
+ * make room for the GUARANTEED container.
+ */
+ @Test
+ public void testKillOppContainersWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is very high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1800, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ // the last launched OPPORTUNISTIC container is expected to be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(1))) {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ } else {
+ Assert.assertEquals(status.getContainerId() + " is not RUNNING",
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+
+ /**
+ * Start three OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node, yet whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization.
+ * Then try to start a GUARANTEED container. Even though the node has
+ * nothing left unallocated, it is expected to start immediately
+ * without killing any running OPPORTUNISTIC containers because the node
+ * utilization is very low and overallocation is turned on.
+ */
+ @Test
+ public void testKillNoOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All three GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1024, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start four OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node. The real resource utilization of the first two
+ * OPPORTUNISTIC containers are high whereas that of the latter two are
+ * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
+ * container will eventually start running after preempting the third
+ * and fourth OPPORTUNISTIC container (which releases no resources) and
+ * then the second OPPORTUNISTIC container.
+ */
+ @Test
+ public void
+ testKillOppContainersConservativelyWithOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All four GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ // the containers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start a GUARANTEED container when there's nothing left unallocated
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(4,
+ BuilderUtils.newResource(1024, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ add(createContainerId(3));
+ add(createContainerId(4));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(0)) ||
+ status.getContainerId().equals(createContainerId(4))) {
+ Assert.assertEquals(
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ } else {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
+ * which in aggregate exceeds the capacity of the node. The first two
+ * OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
+ * one utilizes nearly all of its resource requested. Then try to start two
+ * more OPPORTUNISTIC containers. The two OPPORTUNISTIC containers are
+ * expected to be queued immediately. Upon the completion of the
+ * resource-usage-heavy GUARANTEED container, both OPPORTUNISTIC containers
+ * are expected to start.
+ */
+ @Test
+ public void testStartOppContainersUponContainerCompletion() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+
+ // All three containers are all expected to start immediately
+ // because the node utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the contianers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(4,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ // the two new OPPORTUNISTIC containers are expected to be queued
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(3), ContainerSubState.SCHEDULED);
+ put(createContainerId(4), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+ allowContainerToSucceed(2);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.DONE);
+
+ // the two OPPORTUNISTIC containers are expected to start together
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.DONE);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ put(createContainerId(4), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED container that consumes all the resources on the
+ * node and keeps running, followed by two OPPORTUNISTIC containers that
+ * will be queued forever because there is no containers starting or
+ * finishing. Then try to start OPPORTUNISTIC containers out of band.
+ */
+ @Test
+ public void testStartOpportunisticContainersOutOfBand() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(0,
+ BuilderUtils.newResource(2048, 4), true))));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+
+ // the container is fully utilizing its resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+
+ // send two OPPORTUNISTIC container requests that are expected to be queued
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.SCHEDULED);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ // the containers utilization dropped to the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start containers out of band.
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // no containers in queue are expected to be launched because the
+ // containers utilization is not below the over-allocation threshold
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.SCHEDULED);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+
+ // the containers utilization dropped way below the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // the two OPPORTUNISTIC containers are expected to be launched
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ private void setContainerResourceUtilization(ResourceUtilization usage) {
+ ((ContainerMonitorForOverallocationTest)
+ containerManager.getContainersMonitor())
+ .setContainerResourceUsage(usage);
+ }
+
+ private void allowContainerToSucceed(int containerId) {
+ ((LongRunningContainerSimulatingContainerExecutor) this.exec)
+ .containerSucceeded(createContainerId(containerId));
+ }
+
+
+ protected StartContainerRequest createStartContainerRequest(int containerId,
+ Resource resource, boolean isGuaranteed) throws IOException {
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
+ ExecutionType.OPPORTUNISTIC;
+ Token containerToken = createContainerToken(
+ createContainerId(containerId),
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
+ context.getContainerTokenSecretManager(),
+ null, executionType);
+
+ return StartContainerRequest.newInstance(
+ containerLaunchContext, containerToken);
+ }
+
+ protected void verifyContainerStatuses(
+ Map<ContainerId, ContainerSubState> expected)
+ throws IOException, YarnException {
+ List<ContainerId> statList = new ArrayList<>(expected.keySet());
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+
+ for (ContainerStatus status : containerStatuses) {
+ ContainerId containerId = status.getContainerId();
+ Assert.assertEquals(containerId + " is in unexpected state",
+ expected.get(containerId), status.getContainerSubState());
+ }
+ }
+
+ /**
+ * A container manager that sends a dummy container pid while it's cleaning
+ * up running containers. Used along with
+ * LongRunningContainerSimulatingContainerExecutor to simulate long running
+ * container processes for testing purposes.
+ */
+ private static class LongRunningContainerSimulatingContainersManager
+ extends ContainerManagerImpl {
+
+ private final String user;
+
+ LongRunningContainerSimulatingContainersManager(
+ Context context, ContainerExecutor exec,
+ DeletionService deletionContext,
+ NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler, String user) {
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler);
+ this.user = user;
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ /**
+ * Create a container launcher that signals container processes
+ * with a dummy pid. The container processes are simulated in
+ * LongRunningContainerSimulatingContainerExecutor which does
+ * not write a pid file on behalf of containers to launch, so
+ * the pid does not matter.
+ */
+ @Override
+ protected ContainersLauncher createContainersLauncher(
+ Context context, ContainerExecutor exec) {
+ ContainerManagerImpl containerManager = this;
+ return new ContainersLauncher(context, dispatcher, exec, dirsHandler,
+ this) {
+ @Override
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager) {
+ @Override
+ protected String getContainerPid(Path pidFilePath)
+ throws Exception {
+ return "123";
+ }
+
+ };
+ }
+ };
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(
+ ContainerExecutor exec) {
+ return new ContainerMonitorForOverallocationTest(exec,
+ dispatcher, context);
+ }
+
+ public void startContainersOutOfBandUponLowUtilization() {
+ ((ContainerMonitorForOverallocationTest) getContainersMonitor())
+ .attemptToStartContainersUponLowUtilization();
+ }
+ }
+
+ /**
+ * A container executor that simulates long running container processes
+ * by having container launch threads sleep infinitely until it's given
+ * a signal to finish with either a success or failure exit code.
+ */
+ private static class LongRunningContainerSimulatingContainerExecutor
+ extends DefaultContainerExecutor {
+ private ConcurrentHashMap<ContainerId, ContainerFinishLatch> containers =
+ new ConcurrentHashMap<>();
+
+ public void containerSucceeded(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toSucceed();
+ }
+ }
+
+ public void containerFailed(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toFail();
+ }
+ }
+
+ /**
+ * Simulate long running container processes by having container launcher
+ * threads wait infinitely for a signal to finish.
+ */
+ @Override
+ public int launchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ ContainerId container = ctx.getContainer().getContainerId();
+ containers.putIfAbsent(container, new ContainerFinishLatch(container));
+
+ // simulate a long running container process by having the
+ // container launch thread sleep forever until it's given a
+ // signal to finish with a exit code.
+ while (!containers.get(container).toProceed) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ return -1;
+ }
+ }
+
+ return containers.get(container).getContainerExitCode();
+ }
+
+ /**
+ * Override signalContainer() so that simulated container processes
+ * are properly cleaned up.
+ */
+ @Override
+ public boolean signalContainer(ContainerSignalContext ctx)
+ throws IOException {
+ containerSucceeded(ctx.getContainer().getContainerId());
+ return true;
+ }
+
+ /**
+ * A signal that container launch threads wait for before exiting
+ * in order to simulate long running container processes.
+ */
+ private static final class ContainerFinishLatch {
+ volatile boolean toProceed;
+ int exitCode;
+ ContainerId container;
+
+ ContainerFinishLatch(ContainerId containerId) {
+ exitCode = 0;
+ toProceed = false;
+ container = containerId;
+ }
+
+ void toSucceed() {
+ exitCode = 0;
+ toProceed = true;
+ }
+
+ void toFail() {
+ exitCode = -101;
+ toProceed = true;
+ }
+
+ int getContainerExitCode() {
+ // read barrier of toProceed to make sure the exit code is not stale
+ if (toProceed) {
+ LOG.debug(container + " finished with exit code: " + exitCode);
+ }
+ return exitCode;
+ }
+ }
+ }
+
+ /**
+ * A test implementation of container monitor that allows control of
+ * current resource utilization.
+ */
+ private static class ContainerMonitorForOverallocationTest
+ extends ContainersMonitorImpl {
+
+ private ResourceUtilization containerResourceUsage =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
+ ContainerMonitorForOverallocationTest(ContainerExecutor exec,
+ AsyncDispatcher dispatcher, Context context) {
+ super(exec, dispatcher, context);
+ }
+
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
+ }
+
+ @Override
+ public long getVmemAllocatedForContainers() {
+ float pmemRatio = getConfig().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ return (long) (pmemRatio * getPmemAllocatedForContainers());
+ }
+
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return NM_CONTAINERS_VCORES;
+ }
+
+ @Override
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ return new ContainersMonitor.ContainersResourceUtilization(
+ containerResourceUsage, System.currentTimeMillis());
+ }
+
+ public void setContainerResourceUsage(
+ ResourceUtilization containerResourceUsage) {
+ this.containerResourceUsage = containerResourceUsage;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-8427. Don't start
opportunistic containers at container scheduler/finish event with
over-allocation. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-8427. Don't start opportunistic containers at container scheduler/finish event with over-allocation. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23ad2fbd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23ad2fbd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23ad2fbd
Branch: refs/heads/YARN-1011
Commit: 23ad2fbd5b151fa0b19b7f224348560b8d29ac83
Parents: 3ff8d3f
Author: Miklos Szegedi <sz...@apache.org>
Authored: Fri Jun 22 10:25:31 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:42:51 2018 -0700
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 6 +-
.../monitor/ContainersMonitorImpl.java | 2 +-
.../AllocationBasedResourceTracker.java | 8 +-
.../scheduler/ContainerScheduler.java | 38 +++---
.../scheduler/ResourceUtilizationTracker.java | 6 +
.../UtilizationBasedResourceTracker.java | 2 +-
...estContainerSchedulerWithOverAllocation.java | 126 +++++++++++++++++--
7 files changed, 155 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index a08e227..0850f74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -236,7 +236,7 @@ public class ContainerManagerImpl extends CompositeService implements
this.dirsHandler = dirsHandler;
// ContainerManager level dispatcher.
- dispatcher = new AsyncDispatcher("NM ContainerManager dispatcher");
+ dispatcher = createDispatcher();
this.deletionService = deletionContext;
this.metrics = metrics;
@@ -295,6 +295,10 @@ public class ContainerManagerImpl extends CompositeService implements
this.writeLock = lock.writeLock();
}
+ protected AsyncDispatcher createDispatcher() {
+ return new AsyncDispatcher("NM ContainerManager dispatcher");
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a045d78..8f72b56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -271,7 +271,7 @@ public class ContainersMonitorImpl extends AbstractService implements
/**
* Check all prerequisites for NM over-allocation.
*/
- private void checkOverAllocationPrerequisites() throws YarnException {
+ protected void checkOverAllocationPrerequisites() throws YarnException {
// LinuxContainerExecutor is required to enable overallocation
if (!(containerExecutor instanceof LinuxContainerExecutor)) {
throw new YarnException(LinuxContainerExecutor.class.getName() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
index 86b3698..a3e19eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -56,11 +56,8 @@ public class AllocationBasedResourceTracker
return this.containersAllocation;
}
- /**
- * Get the amount of resources that have not been allocated to containers.
- * @return Resource resources that have not been allocated to containers.
- */
- protected Resource getUnallocatedResources() {
+ @Override
+ public Resource getUnallocatedResources() {
// unallocated resources = node capacity - containers allocation
// = -(container allocation - node capacity)
ResourceUtilization allocationClone =
@@ -80,7 +77,6 @@ public class AllocationBasedResourceTracker
return unallocated;
}
-
@Override
public Resource getAvailableResources() {
return getUnallocatedResources();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0bebe44..0d7de67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -218,7 +218,9 @@ public class ContainerScheduler extends AbstractService implements
startPendingContainers(false);
break;
case SCHEDULE_CONTAINERS:
- startPendingContainers(true);
+ // try to launch OPPORTUNISTIC containers allowing over-allocation if the
+ // node utilization is low.
+ startOpportunisticContainers(utilizationTracker.getAvailableResources());
break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
@@ -410,11 +412,7 @@ public class ContainerScheduler extends AbstractService implements
this.metrics.completeOpportunisticContainer(container.getResource());
}
- // In case of over-allocation being turned on, we may need to reclaim
- // more resources since the opportunistic containers that have been
- // killed or paused may have not released as much resource as we need.
- boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
- startPendingContainers(reclaimOpportunisticResources);
+ startPendingContainers(false);
}
}
@@ -434,22 +432,32 @@ public class ContainerScheduler extends AbstractService implements
return;
}
+ boolean guaranteedContainersLaunched = startGuaranteedContainers(
+ reclaimOpportunisticResources);
+
+ if (guaranteedContainersLaunched) {
+ // if all GUARANTEED containers are launched, try to launch OPPORTUNISTIC
+ // containers with unallocated resources only.
+ startOpportunisticContainers(
+ utilizationTracker.getUnallocatedResources());
+ }
+ }
+
+ private boolean startGuaranteedContainers(
+ boolean reclaimOpportunisticResources) {
Resource available = utilizationTracker.getAvailableResources();
// Start guaranteed containers that are queued, if resources available.
boolean allGuaranteedContainersLaunched =
startGuaranteedContainers(available);
- // Start opportunistic containers, if resources available, which is true
- // if all guaranteed containers in queue have been launched.
- if (allGuaranteedContainersLaunched) {
- startOpportunisticContainers(available);
- } else {
+
+ if (!allGuaranteedContainersLaunched && reclaimOpportunisticResources) {
// If not all guaranteed containers in queue are launched, we may need
// to reclaim resources from opportunistic containers that are running.
- if (reclaimOpportunisticResources) {
- reclaimOpportunisticContainerResources();
- }
+ reclaimOpportunisticContainerResources();
}
+
+ return allGuaranteedContainersLaunched;
}
/**
@@ -693,8 +701,6 @@ public class ContainerScheduler extends AbstractService implements
*
* If the node is over-allocating itself, this may cause not enough
* OPPORTUNISTIC containers being killed/paused in cases where the running
- * OPPORTUNISTIC containers are not consuming fully their resource requests.
- * We'd check again upon container completion events to see if more running
* OPPORTUNISTIC containers need to be killed/paused.
*
* @return the amount of resource needed to be reclaimed for this container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 98d99c6..7a7c78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -45,6 +45,12 @@ public interface ResourceUtilizationTracker {
Resource getAvailableResources();
/**
+ * Get the amount of resources left un-allocated.
+ * @return Resource unallocated resources
+ */
+ Resource getUnallocatedResources();
+
+ /**
* Add Container's resources to Node Utilization upon container launch.
* @param container Container.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
index 6f9bc82..f976700 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
@@ -61,7 +61,7 @@ public class UtilizationBasedResourceTracker
@Override
public Resource getAvailableResources() {
- Resource resourceBasedOnAllocation = getUnallocatedResources();
+ Resource resourceBasedOnAllocation = super.getUnallocatedResources();
Resource resourceBasedOnUtilization =
getResourcesAvailableBasedOnUtilization();
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ad2fbd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
index 384b116..183d868 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
@@ -114,6 +115,9 @@ public class TestContainerSchedulerWithOverAllocation
conf.setFloat(
YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
0.75f);
+ // disable the monitor thread in ContainersMonitor to allow control over
+ // when opportunistic containers are launched with over-allocation
+ conf.setBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false);
super.setup();
}
@@ -190,6 +194,27 @@ public class TestContainerSchedulerWithOverAllocation
BuilderUtils.newResource(512, 1), false))
));
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // this container is not expected to be started immediately because
+ // opportunistic containers cannot be started if the node would be
+ // over-allocated
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // try to start opportunistic containers out of band.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
// this container is expected to be started immediately because there
// are (memory: 1024, vcore: 0.625) available based on over-allocation
BaseContainerManagerTest.waitForContainerSubState(
@@ -244,6 +269,16 @@ public class TestContainerSchedulerWithOverAllocation
createStartContainerRequest(2,
BuilderUtils.newResource(512, 1), false))
));
+
+ // try to start opportunistic containers out of band because they can
+ // not be launched at container scheduler event if the node would be
+ // over-allocated.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
// this container will not start immediately because there is not
// enough resource available at the moment either in terms of
// resources unallocated or in terms of the actual utilization
@@ -299,6 +334,27 @@ public class TestContainerSchedulerWithOverAllocation
BuilderUtils.newResource(512, 1), false))
));
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // this container is not expected to be started immediately because
+ // opportunistic containers cannot be started if the node would be
+ // over-allocated
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // try to start opportunistic containers out of band.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
// this container is expected to be started because there is resources
// available because the actual utilization is very low
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -355,6 +411,9 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(
StartContainersRequest.newInstance(moreContainerRequests));
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
// All OPPORTUNISTIC containers but the last one should be queued.
// The last OPPORTUNISTIC container to launch should be killed.
BaseContainerManagerTest.waitForContainerState(
@@ -580,13 +639,27 @@ public class TestContainerSchedulerWithOverAllocation
}
}
));
- // All three GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // Two OPPORTUNISTIC containers are expected to start with the
+ // unallocated resources, but one will be queued because no
+ // over-allocation is allowed at container scheduler events.
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(0), ContainerSubState.RUNNING);
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(1), ContainerSubState.RUNNING);
BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ // try to start the opportunistic container out of band because it can
+ // not be launched at container scheduler event if the node would be
+ // over-allocated.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // now the queued opportunistic container should also start
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.RUNNING);
// the containers utilization is low
@@ -625,7 +698,6 @@ public class TestContainerSchedulerWithOverAllocation
* and fourth OPPORTUNISTIC container (which releases no resources) and
* then the second OPPORTUNISTIC container.
*/
- @Test
public void
testKillOppContainersConservativelyWithOverallocationHighUtilization()
throws Exception {
@@ -730,7 +802,7 @@ public class TestContainerSchedulerWithOverAllocation
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.RUNNING);
- // the contianers utilization is at the overallocation threshold
+ // the container utilization is at the overallocation threshold
setContainerResourceUtilization(
ResourceUtilization.newInstance(1536, 0, 1.0f/2));
@@ -740,7 +812,7 @@ public class TestContainerSchedulerWithOverAllocation
add(createStartContainerRequest(3,
BuilderUtils.newResource(512, 1), false));
add(createStartContainerRequest(4,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(800, 1), false));
}
}
));
@@ -759,12 +831,35 @@ public class TestContainerSchedulerWithOverAllocation
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.DONE);
- // the two OPPORTUNISTIC containers are expected to start together
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // only one OPPORTUNISTIC container is start because no over-allocation
+ // is allowed to start OPPORTUNISTIC containers at container finish event.
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(3), ContainerSubState.RUNNING);
BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
+ createContainerId(4), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.DONE);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ put(createContainerId(4), ContainerSubState.SCHEDULED);
+ }
+ });
+ // now try to start the OPPORTUNISTIC container that was queued because
+ // we don't start OPPORTUNISTIC containers at container finish event if
+ // the node would be over-allocated
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
{
put(createContainerId(0), ContainerSubState.RUNNING);
@@ -817,7 +912,7 @@ public class TestContainerSchedulerWithOverAllocation
setContainerResourceUtilization(
ResourceUtilization.newInstance(1536, 0, 1.0f/2));
- // try to start containers out of band.
+ // try to start opportunistic containers out of band.
((LongRunningContainerSimulatingContainersManager)containerManager)
.startContainersOutOfBandUponLowUtilization();
@@ -937,6 +1032,11 @@ public class TestContainerSchedulerWithOverAllocation
return ugi;
}
+ @Override
+ protected AsyncDispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+
/**
* Create a container launcher that signals container processes
* with a dummy pid. The container processes are simulated in
@@ -977,6 +1077,10 @@ public class TestContainerSchedulerWithOverAllocation
((ContainerMonitorForOverallocationTest) getContainersMonitor())
.attemptToStartContainersUponLowUtilization();
}
+
+ public void drainAsyncEvents() {
+ ((DrainDispatcher) dispatcher).await();
+ }
}
/**
@@ -1113,6 +1217,12 @@ public class TestContainerSchedulerWithOverAllocation
containerResourceUsage, System.currentTimeMillis());
}
+ @Override
+ protected void checkOverAllocationPrerequisites() {
+ // do not check
+ }
+
+
public void setContainerResourceUsage(
ResourceUtilization containerResourceUsage) {
this.containerResourceUsage = containerResourceUsage;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-7491. Make sure AM is not
scheduled on an opportunistic container. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/979420f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/979420f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/979420f4
Branch: refs/heads/YARN-1011
Commit: 979420f4189a0e4ad371a69013f7f7ebd1febf55
Parents: 353bdbc
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:39:11 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++-
.../yarn/server/utils/TestBuilderUtils.java | 54 +++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 73 ++++++++++++++++----
4 files changed, 129 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index ff88225..a507e8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -351,15 +351,28 @@ public class BuilderUtils {
return priority;
}
+ public static ResourceRequest newAMResourceRequest(Priority priority,
+ String hostName, Resource capability) {
+ return newResourceRequest(priority, hostName, capability, 1,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+ }
+
public static ResourceRequest newResourceRequest(Priority priority,
String hostName, Resource capability, int numContainers) {
+ return newResourceRequest(priority, hostName, capability,
+ numContainers, ExecutionTypeRequest.newInstance());
+ }
+
+ public static ResourceRequest newResourceRequest(Priority priority,
+ String hostName, Resource capability, int numContainers,
+ ExecutionTypeRequest executionTypeRequest) {
ResourceRequest request = recordFactory
.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
- request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+ request.setExecutionTypeRequest(executionTypeRequest);
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+ @Test
+ public void testNewAMResourceRequest() {
+ final Priority priority = Priority.newInstance(0);
+ final String hostName = "ResourceA";
+ final Resource capacity = Resource.newInstance(1024, 1);
+
+ ResourceRequest resourceRequest =
+ BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+ Assert.assertEquals("unexpected priority", priority,
+ resourceRequest.getPriority());
+ Assert.assertEquals("unexpected host name", hostName,
+ resourceRequest.getResourceName());
+ Assert.assertEquals("unexpected capacity", capacity,
+ resourceRequest.getCapability());
+ Assert.assertEquals("unexpected number of containers", 1,
+ resourceRequest.getNumContainers());
+ Assert.assertEquals("unexpected execution type",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+ resourceRequest.getExecutionTypeRequest());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index d0f2ce6..6ea55ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -516,8 +516,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (amReqs == null || amReqs.isEmpty()) {
if (submissionContext.getResource() != null) {
amReqs = Collections.singletonList(BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1));
+ .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource()));
} else {
throw new InvalidResourceRequestException("Invalid resource request, "
+ "no resources requested");
@@ -548,7 +548,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
for (ResourceRequest amReq : amReqs) {
amReq.setCapability(anyReq.getCapability());
amReq.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
amReq.setNumContainers(1);
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 0bd5372..8617dc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -839,15 +839,45 @@ public class TestAppManager{
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequest(
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build());
List<ResourceRequest> reqs = new ArrayList<>();
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack", Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack/node", Resources.createResource(1025), 1, true));
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack/node")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
// getAMContainerResourceRequest uses the first entry of
// getAMContainerResourceRequests
@@ -869,9 +899,15 @@ public class TestAppManager{
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
ResourceRequest req =
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, true);
- req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression(RMNodeLabelsManager.NO_LABEL)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build();
asContext.setAMContainerResourceRequest(ResourceRequest.clone(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
@@ -892,9 +928,16 @@ public class TestAppManager{
// setResource
Assert.assertEquals(Collections.singletonList(
- ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, Resources.createResource(1024), 1, true,
- "")),
+ ResourceRequest.newBuilder()
+ .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression("")
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()),
app.getAMResourceRequests());
}
@@ -936,7 +979,7 @@ public class TestAppManager{
for (ResourceRequest req : reqs) {
req.setCapability(anyReq.getCapability());
req.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
req.setNumContainers(1);
req.setPriority(Priority.newInstance(0));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-13892. Disk Balancer: Make
execute command documentation better. Contributed by Ranith Sardar.
Posted by ha...@apache.org.
HDFS-13892. Disk Balancer: Make execute command documentation better.
Contributed by Ranith Sardar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fc293fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fc293fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fc293fe
Branch: refs/heads/YARN-1011
Commit: 6fc293fece935e3524ae59699aa3c3e3d98f6d86
Parents: 6b5838e
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 19 20:48:41 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 19 20:48:41 2018 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fc293fe/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
index 5dd6ffc..955f179 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
@@ -78,7 +78,9 @@ Execute command takes a plan command executes it against the datanode that plan
`hdfs diskbalancer -execute /system/diskbalancer/nodename.plan.json`
This executes the plan by reading datanode’s address from the plan file.
-
+When DiskBalancer executes the plan, it is the beginning of an asynchronous process that can take a long time.
+So, query command can help to get the current status of execute command.
+
| COMMAND\_OPTION | Description |
|:---- |:---- |
| `-skipDateCheck` | Skip date check and force execute the plan.|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-15748. S3 listing
inconsistency can raise NPE in globber. Contributed by Steve Loughran.
Posted by ha...@apache.org.
HADOOP-15748. S3 listing inconsistency can raise NPE in globber.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/646874c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/646874c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/646874c3
Branch: refs/heads/YARN-1011
Commit: 646874c326139457b79cf8cfa547b3c91a78c7b4
Parents: 7ad27e9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 20 13:04:52 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 20 13:04:52 2018 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/Globber.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/646874c3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index ca3db1d..b241a94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -245,7 +245,18 @@ class Globber {
// incorrectly conclude that /a/b was a file and should not match
// /a/*/*. So we use getFileStatus of the path we just listed to
// disambiguate.
- if (!getFileStatus(candidate.getPath()).isDirectory()) {
+ Path path = candidate.getPath();
+ FileStatus status = getFileStatus(path);
+ if (status == null) {
+ // null means the file was not found
+ LOG.warn("File/directory {} not found:"
+ + " it may have been deleted."
+ + " If this is an object store, this can be a sign of"
+ + " eventual consistency problems.",
+ path);
+ continue;
+ }
+ if (!status.isDirectory()) {
continue;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: Revert "Revert "YARN-6675. Add NM
support to launch opportunistic containers based on overallocation.
Contributed by Haibo Chen.""
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index 6b3ac67..f24df38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -66,8 +67,8 @@ public class TestContainerSchedulerRecovery {
@Mock private ContainerId containerId;
- @Mock private AllocationBasedResourceUtilizationTracker
- allocationBasedResourceUtilizationTracker;
+ @Mock private AllocationBasedResourceTracker
+ allocationBasedResourceTracker;
@InjectMocks private ContainerScheduler tempContainerScheduler =
new ContainerScheduler(context, dispatcher, metrics, 0);
@@ -85,12 +86,13 @@ public class TestContainerSchedulerRecovery {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
when(container.getContainerId()).thenReturn(containerId);
+ when(container.getResource()).thenReturn(Resource.newInstance(1024, 1));
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(containerId.getApplicationAttemptId().getApplicationId())
.thenReturn(appId);
when(containerId.getContainerId()).thenReturn(123L);
- doNothing().when(allocationBasedResourceUtilizationTracker)
- .addContainerResources(container);
+ doNothing().when(allocationBasedResourceTracker)
+ .containerLaunched(container);
}
@After public void tearDown() {
@@ -112,8 +114,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
@@ -132,8 +134,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, GUARANTEED,
@@ -152,8 +154,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
@@ -172,8 +174,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, GUARANTEED,
@@ -192,8 +194,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
@@ -212,8 +214,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, GUARANTEED,
@@ -232,8 +234,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
@@ -252,8 +254,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, GUARANTEED,
@@ -272,8 +274,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
@@ -292,8 +294,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as GUARANTEED but no executionType set,
@@ -311,8 +313,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED but no executionType set,
@@ -330,7 +332,7 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/faa71dbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
new file mode 100644
index 0000000..384b116
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -0,0 +1,1121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+/**
+ * Test ContainerScheduler behaviors when NM overallocation is turned on.
+ */
+public class TestContainerSchedulerWithOverAllocation
+ extends BaseContainerManagerTest {
+ private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
+ private static final int NM_CONTAINERS_VCORES = 4;
+ private static final int NM_CONTAINERS_MEMORY_MB = 2048;
+
+ static {
+ LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
+ }
+
+ public TestContainerSchedulerWithOverAllocation()
+ throws UnsupportedFileSystemException {
+ }
+
+ @Override
+ protected ContainerExecutor createContainerExecutor() {
+ DefaultContainerExecutor exec =
+ new LongRunningContainerSimulatingContainerExecutor();
+ exec.setConf(conf);
+ return exec;
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(
+ DeletionService delSrvc) {
+ return new LongRunningContainerSimulatingContainersManager(
+ context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler, user);
+ }
+
+ @Override
+ public void setup() throws IOException {
+ conf.setInt(
+ YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
+ NM_OPPORTUNISTIC_QUEUE_LIMIT);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
+ 0.75f);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
+ 0.75f);
+ super.setup();
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC container, which in aggregate do
+ * not exceed the capacity of the node. Both containers are expected to start
+ * running immediately.
+ */
+ @Test
+ public void testStartMultipleContainersWithoutOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ StartContainersRequest allRequests = StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() { {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ } }
+ );
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers whose utilization
+ * is very low relative to their resource request, resulting in a low node
+ * utilization. Then start another OPPORTUNISTIC containers which requests
+ * more than what's left unallocated on the node. Due to overallocation
+ * being turned on and node utilization being low, the second OPPORTUNISTIC
+ * container is also expected to be launched immediately.
+ */
+ @Test
+ public void testStartOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started immediately because there
+ // are (memory: 1024, vcore: 0.625) available based on over-allocation
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers which utilizes most
+ * of the resources they requested, resulting in a high node utilization.
+ * Then start another OPPORTUNISTIC containers which requests more than what's
+ * left unallocated on the node. Because of the high resource utilization on
+ * the node, the projected utilization, if we were to start the second
+ * OPPORTUNISTIC container immediately, will go over the NM overallocation
+ * threshold, so the second OPPORTUNISTIC container is expected to be queued.
+ */
+ @Test
+ public void testQueueOppContainerWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1500, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+ // this container will not start immediately because there is not
+ // enough resource available at the moment either in terms of
+ // resources unallocated or in terms of the actual utilization
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+ }
+
+ /**
+ * Start two GUARANTEED containers which in aggregate takes up the whole node
+ * capacity, yet whose utilization is low relative to their resource request,
+ * resulting in a low node resource utilization. Then try to start another
+ * OPPORTUNISTIC containers. Because the resource utilization across the node
+ * is low and overallocation being turned on, the OPPORTUNISTIC container is
+ * expected to be launched immediately even though there is no resources left
+ * unallocated.
+ */
+ @Test
+ public void testStartOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(800, 0, 1.0f/8));
+
+ // start a container when there is no resources left unallocated.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started because there is resources
+ // available because the actual utilization is very low
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ /**
+ * Start two GUARANTEED containers which in aggregate take up the whole node
+ * capacity and fully utilize the resources they requested. Then try to start
+ * four OPPORTUNISTIC containers of which three will be queued and one will be
+ * killed because of the max queue length is 3.
+ */
+ @Test
+ public void testQueueOppContainersWithFullUtilization() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers are fully utilizing their resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f/8));
+
+ // start more OPPORTUNISTIC containers than what the OPPORTUNISTIC container
+ // queue can hold when there is no unallocated resource left.
+ List<StartContainerRequest> moreContainerRequests =
+ new ArrayList<>(NM_OPPORTUNISTIC_QUEUE_LIMIT + 1);
+ for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
+ moreContainerRequests.add(
+ createStartContainerRequest(2 + a,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ containerManager.startContainers(
+ StartContainersRequest.newInstance(moreContainerRequests));
+
+ // All OPPORTUNISTIC containers but the last one should be queued.
+ // The last OPPORTUNISTIC container to launch should be killed.
+ BaseContainerManagerTest.waitForContainerState(
+ containerManager, createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT + 2),
+ ContainerState.COMPLETE);
+
+ HashMap<ContainerId, ContainerSubState> expectedContainerStatus =
+ new HashMap<>();
+ expectedContainerStatus.put(
+ createContainerId(0), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(1), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT),
+ ContainerSubState.DONE);
+ for (int i = 0; i < NM_OPPORTUNISTIC_QUEUE_LIMIT; i++) {
+ expectedContainerStatus.put(
+ createContainerId(i + 2), ContainerSubState.SCHEDULED);
+ }
+ verifyContainerStatuses(expectedContainerStatus);
+ }
+
+ /**
+ * Start two GUARANTEED containers that together does not take up the
+ * whole node. Then try to start one OPPORTUNISTIC container that will
+ * fit into the remaining unallocated space on the node.
+ * The OPPORTUNISTIC container is expected to start even though the
+ * current node utilization is above the NM overallocation threshold,
+ * because it's always safe to launch containers as long as the node
+ * has not been fully allocated.
+ */
+ @Test
+ public void testStartOppContainerWithHighUtilizationNoOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1200, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(400, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is above the over-allocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1600, 0, 1.0f/2));
+
+ // start a container that can just fit in the remaining unallocated space
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(400, 1), false))
+ ));
+
+ // the OPPORTUNISTIC container can be safely launched even though
+ // the container utilization is above the NM overallocation threshold
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers first whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization. Then
+ * try to start a GUARANTEED container which requests more than what's left
+ * unallocated on the node. Because the node utilization is low and NM
+ * overallocation is turned on, the GUARANTEED container is expected to be
+ * started immediately without killing any running OPPORTUNISTIC containers.
+ */
+ @Test
+ public void testKillNoOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers.
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers whose utilization will be high relative
+ * to the resources they requested, resulting in a high node utilization.
+ * Then try to start a GUARANTEED container which requests more than what's
+ * left unallocated on the node. Because the node is under high utilization,
+ * the second OPPORTUNISTIC container is expected to be killed in order to
+ * make room for the GUARANTEED container.
+ */
+ @Test
+ public void testKillOppContainersWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is very high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1800, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ // the last launched OPPORTUNISTIC container is expected to be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(1))) {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ } else {
+ Assert.assertEquals(status.getContainerId() + " is not RUNNING",
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+
+ /**
+ * Start three OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node, yet whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization.
+ * Then try to start a GUARANTEED container. Even though the node has
+ * nothing left unallocated, it is expected to start immediately
+ * without killing any running OPPORTUNISTIC containers because the node
+ * utilization is very low and overallocation is turned on.
+ */
+ @Test
+ public void testKillNoOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All three GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1024, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start four OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node. The real resource utilization of the first two
+ * OPPORTUNISTIC containers are high whereas that of the latter two are
+ * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
+ * container will eventually start running after preempting the third
+ * and fourth OPPORTUNISTIC container (which releases no resources) and
+ * then the second OPPORTUNISTIC container.
+ */
+ @Test
+ public void
+ testKillOppContainersConservativelyWithOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All four GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ // the containers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start a GUARANTEED container when there's nothing left unallocated
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(4,
+ BuilderUtils.newResource(1024, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ add(createContainerId(3));
+ add(createContainerId(4));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(0)) ||
+ status.getContainerId().equals(createContainerId(4))) {
+ Assert.assertEquals(
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ } else {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
+ * which in aggregate exceeds the capacity of the node. The first two
+ * OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
+ * one utilizes nearly all of its resource requested. Then try to start two
+ * more OPPORTUNISTIC containers. The two OPPORTUNISTIC containers are
+ * expected to be queued immediately. Upon the completion of the
+ * resource-usage-heavy GUARANTEED container, both OPPORTUNISTIC containers
+ * are expected to start.
+ */
+ @Test
+ public void testStartOppContainersUponContainerCompletion() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+
+ // All three containers are all expected to start immediately
+ // because the node utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the contianers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(4,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ // the two new OPPORTUNISTIC containers are expected to be queued
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(3), ContainerSubState.SCHEDULED);
+ put(createContainerId(4), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+ allowContainerToSucceed(2);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.DONE);
+
+ // the two OPPORTUNISTIC containers are expected to start together
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.DONE);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ put(createContainerId(4), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED container that consumes all the resources on the
+ * node and keeps running, followed by two OPPORTUNISTIC containers that
+ * will be queued forever because there is no containers starting or
+ * finishing. Then try to start OPPORTUNISTIC containers out of band.
+ */
+ @Test
+ public void testStartOpportunisticContainersOutOfBand() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(0,
+ BuilderUtils.newResource(2048, 4), true))));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+
+ // the container is fully utilizing its resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+
+ // send two OPPORTUNISTIC container requests that are expected to be queued
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.SCHEDULED);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ // the containers utilization dropped to the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start containers out of band.
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // no containers in queue are expected to be launched because the
+ // containers utilization is not below the over-allocation threshold
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.SCHEDULED);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+
+ // the containers utilization dropped way below the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // the two OPPORTUNISTIC containers are expected to be launched
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ private void setContainerResourceUtilization(ResourceUtilization usage) {
+ ((ContainerMonitorForOverallocationTest)
+ containerManager.getContainersMonitor())
+ .setContainerResourceUsage(usage);
+ }
+
+ private void allowContainerToSucceed(int containerId) {
+ ((LongRunningContainerSimulatingContainerExecutor) this.exec)
+ .containerSucceeded(createContainerId(containerId));
+ }
+
+
+ protected StartContainerRequest createStartContainerRequest(int containerId,
+ Resource resource, boolean isGuaranteed) throws IOException {
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
+ ExecutionType.OPPORTUNISTIC;
+ Token containerToken = createContainerToken(
+ createContainerId(containerId),
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
+ context.getContainerTokenSecretManager(),
+ null, executionType);
+
+ return StartContainerRequest.newInstance(
+ containerLaunchContext, containerToken);
+ }
+
+ protected void verifyContainerStatuses(
+ Map<ContainerId, ContainerSubState> expected)
+ throws IOException, YarnException {
+ List<ContainerId> statList = new ArrayList<>(expected.keySet());
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+
+ for (ContainerStatus status : containerStatuses) {
+ ContainerId containerId = status.getContainerId();
+ Assert.assertEquals(containerId + " is in unexpected state",
+ expected.get(containerId), status.getContainerSubState());
+ }
+ }
+
+ /**
+ * A container manager that sends a dummy container pid while it's cleaning
+ * up running containers. Used along with
+ * LongRunningContainerSimulatingContainerExecutor to simulate long running
+ * container processes for testing purposes.
+ */
+ private static class LongRunningContainerSimulatingContainersManager
+ extends ContainerManagerImpl {
+
+ private final String user;
+
+ LongRunningContainerSimulatingContainersManager(
+ Context context, ContainerExecutor exec,
+ DeletionService deletionContext,
+ NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler, String user) {
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler);
+ this.user = user;
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ /**
+ * Create a container launcher that signals container processes
+ * with a dummy pid. The container processes are simulated in
+ * LongRunningContainerSimulatingContainerExecutor which does
+ * not write a pid file on behalf of containers to launch, so
+ * the pid does not matter.
+ */
+ @Override
+ protected ContainersLauncher createContainersLauncher(
+ Context context, ContainerExecutor exec) {
+ ContainerManagerImpl containerManager = this;
+ return new ContainersLauncher(context, dispatcher, exec, dirsHandler,
+ this) {
+ @Override
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager) {
+ @Override
+ protected String getContainerPid(Path pidFilePath)
+ throws Exception {
+ return "123";
+ }
+
+ };
+ }
+ };
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(
+ ContainerExecutor exec) {
+ return new ContainerMonitorForOverallocationTest(exec,
+ dispatcher, context);
+ }
+
+ public void startContainersOutOfBandUponLowUtilization() {
+ ((ContainerMonitorForOverallocationTest) getContainersMonitor())
+ .attemptToStartContainersUponLowUtilization();
+ }
+ }
+
+ /**
+ * A container executor that simulates long running container processes
+ * by having container launch threads sleep infinitely until it's given
+ * a signal to finish with either a success or failure exit code.
+ */
+ private static class LongRunningContainerSimulatingContainerExecutor
+ extends DefaultContainerExecutor {
+ private ConcurrentHashMap<ContainerId, ContainerFinishLatch> containers =
+ new ConcurrentHashMap<>();
+
+ public void containerSucceeded(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toSucceed();
+ }
+ }
+
+ public void containerFailed(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toFail();
+ }
+ }
+
+ /**
+ * Simulate long running container processes by having container launcher
+ * threads wait infinitely for a signal to finish.
+ */
+ @Override
+ public int launchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ ContainerId container = ctx.getContainer().getContainerId();
+ containers.putIfAbsent(container, new ContainerFinishLatch(container));
+
+ // simulate a long running container process by having the
+ // container launch thread sleep forever until it's given a
+ // signal to finish with a exit code.
+ while (!containers.get(container).toProceed) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ return -1;
+ }
+ }
+
+ return containers.get(container).getContainerExitCode();
+ }
+
+ /**
+ * Override signalContainer() so that simulated container processes
+ * are properly cleaned up.
+ */
+ @Override
+ public boolean signalContainer(ContainerSignalContext ctx)
+ throws IOException {
+ containerSucceeded(ctx.getContainer().getContainerId());
+ return true;
+ }
+
+ /**
+ * A signal that container launch threads wait for before exiting
+ * in order to simulate long running container processes.
+ */
+ private static final class ContainerFinishLatch {
+ volatile boolean toProceed;
+ int exitCode;
+ ContainerId container;
+
+ ContainerFinishLatch(ContainerId containerId) {
+ exitCode = 0;
+ toProceed = false;
+ container = containerId;
+ }
+
+ void toSucceed() {
+ exitCode = 0;
+ toProceed = true;
+ }
+
+ void toFail() {
+ exitCode = -101;
+ toProceed = true;
+ }
+
+ int getContainerExitCode() {
+ // read barrier of toProceed to make sure the exit code is not stale
+ if (toProceed) {
+ LOG.debug(container + " finished with exit code: " + exitCode);
+ }
+ return exitCode;
+ }
+ }
+ }
+
+ /**
+ * A test implementation of container monitor that allows control of
+ * current resource utilization.
+ */
+ private static class ContainerMonitorForOverallocationTest
+ extends ContainersMonitorImpl {
+
+ private ResourceUtilization containerResourceUsage =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
+ ContainerMonitorForOverallocationTest(ContainerExecutor exec,
+ AsyncDispatcher dispatcher, Context context) {
+ super(exec, dispatcher, context);
+ }
+
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
+ }
+
+ @Override
+ public long getVmemAllocatedForContainers() {
+ float pmemRatio = getConfig().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ return (long) (pmemRatio * getPmemAllocatedForContainers());
+ }
+
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return NM_CONTAINERS_VCORES;
+ }
+
+ @Override
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ return new ContainersMonitor.ContainersResourceUtilization(
+ containerResourceUsage, System.currentTimeMillis());
+ }
+
+ public void setContainerResourceUsage(
+ ResourceUtilization containerResourceUsage) {
+ this.containerResourceUsage = containerResourceUsage;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org