You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/09/22 20:26:35 UTC
[01/32] hadoop git commit: Update CHANGES.txt to reflect commit of
MR-5982 to 2.7.2
Repository: hadoop
Updated Branches:
refs/heads/YARN-1197 b6fb96980 -> 80c50a42f (forced update)
Update CHANGES.txt to reflect commit of MR-5982 to 2.7.2
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f97dc034
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f97dc034
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f97dc034
Branch: refs/heads/YARN-1197
Commit: f97dc034fe5db5919294465509cfb63d73bcd58e
Parents: 88d89267
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Sep 18 19:24:10 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Sep 18 19:24:10 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f97dc034/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 0c2c8a4..a80a8f6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -565,9 +565,6 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5002. AM could potentially allocate a reduce container to a map
attempt (Chang Li via jlowe)
- MAPREDUCE-5982. Task attempts that fail from the ASSIGNED state can
- disappear (Chang Li via jlowe)
-
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -606,6 +603,9 @@ Release 2.7.2 - UNRELEASED
position/key information for uncompressed input sometimes. (Zhihai Xu via
jlowe)
+ MAPREDUCE-5982. Task attempts that fail from the ASSIGNED state can
+ disappear (Chang Li via jlowe)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[19/32] hadoop git commit: YARN-3866. AM-RM protocol changes to
support container resizing. Contributed by Meng Ding
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
deleted file mode 100644
index 29b0ffe..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
-import org.junit.Test;
-
-public class TestContainerResourceDecrease {
- @Test
- public void testResourceDecreaseContext() {
- ContainerId containerId = ContainerId
- .newContainerId(ApplicationAttemptId.newInstance(
- ApplicationId.newInstance(1234, 3), 3), 7);
- Resource resource = Resource.newInstance(1023, 3);
- ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(
- containerId, resource);
-
- // get proto and recover to ctx
- ContainerResourceDecreaseProto proto =
- ((ContainerResourceDecreasePBImpl) ctx).getProto();
- ctx = new ContainerResourceDecreasePBImpl(proto);
-
- // check values
- Assert.assertEquals(ctx.getCapability(), resource);
- Assert.assertEquals(ctx.getContainerId(), containerId);
- }
-
- @Test
- public void testResourceDecreaseContextWithNull() {
- ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(null,
- null);
-
- // get proto and recover to ctx;
- ContainerResourceDecreaseProto proto =
- ((ContainerResourceDecreasePBImpl) ctx).getProto();
- ctx = new ContainerResourceDecreasePBImpl(proto);
-
- // check values
- Assert.assertNull(ctx.getCapability());
- Assert.assertNull(ctx.getContainerId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
deleted file mode 100644
index 932d5a7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import java.util.Arrays;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
-import org.junit.Test;
-
-public class TestContainerResourceIncrease {
- @Test
- public void testResourceIncreaseContext() {
- byte[] identifier = new byte[] { 1, 2, 3, 4 };
- Token token = Token.newInstance(identifier, "", "".getBytes(), "");
- ContainerId containerId = ContainerId
- .newContainerId(ApplicationAttemptId.newInstance(
- ApplicationId.newInstance(1234, 3), 3), 7);
- Resource resource = Resource.newInstance(1023, 3);
- ContainerResourceIncrease ctx = ContainerResourceIncrease.newInstance(
- containerId, resource, token);
-
- // get proto and recover to ctx
- ContainerResourceIncreaseProto proto =
- ((ContainerResourceIncreasePBImpl) ctx).getProto();
- ctx = new ContainerResourceIncreasePBImpl(proto);
-
- // check values
- Assert.assertEquals(ctx.getCapability(), resource);
- Assert.assertEquals(ctx.getContainerId(), containerId);
- Assert.assertTrue(Arrays.equals(ctx.getContainerToken().getIdentifier()
- .array(), identifier));
- }
-
- @Test
- public void testResourceIncreaseContextWithNull() {
- ContainerResourceIncrease ctx = ContainerResourceIncrease.newInstance(null,
- null, null);
-
- // get proto and recover to ctx;
- ContainerResourceIncreaseProto proto =
- ((ContainerResourceIncreasePBImpl) ctx).getProto();
- ctx = new ContainerResourceIncreasePBImpl(proto);
-
- // check values
- Assert.assertNull(ctx.getContainerToken());
- Assert.assertNull(ctx.getCapability());
- Assert.assertNull(ctx.getContainerId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java
deleted file mode 100644
index cf4dabf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncreaseRequest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreaseRequestPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProto;
-import org.junit.Test;
-
-public class TestContainerResourceIncreaseRequest {
- @Test
- public void ContainerResourceIncreaseRequest() {
- ContainerId containerId = ContainerId
- .newContainerId(ApplicationAttemptId.newInstance(
- ApplicationId.newInstance(1234, 3), 3), 7);
- Resource resource = Resource.newInstance(1023, 3);
- ContainerResourceIncreaseRequest context = ContainerResourceIncreaseRequest
- .newInstance(containerId, resource);
-
- // to proto and get it back
- ContainerResourceIncreaseRequestProto proto =
- ((ContainerResourceIncreaseRequestPBImpl) context).getProto();
- ContainerResourceIncreaseRequest contextRecover =
- new ContainerResourceIncreaseRequestPBImpl(proto);
-
- // check value
- Assert.assertEquals(contextRecover.getContainerId(), containerId);
- Assert.assertEquals(contextRecover.getCapability(), resource);
- }
-
- @Test
- public void testResourceChangeContextWithNullField() {
- ContainerResourceIncreaseRequest context = ContainerResourceIncreaseRequest
- .newInstance(null, null);
-
- // to proto and get it back
- ContainerResourceIncreaseRequestProto proto =
- ((ContainerResourceIncreaseRequestPBImpl) context).getProto();
- ContainerResourceIncreaseRequest contextRecover =
- new ContainerResourceIncreaseRequestPBImpl(proto);
-
- // check value
- Assert.assertNull(contextRecover.getContainerId());
- Assert.assertNull(contextRecover.getCapability());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 6357c36..0979c75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -113,9 +113,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerReport;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
@@ -155,9 +153,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerReportPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreaseRequestPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceChangeRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
@@ -190,9 +186,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerReportProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceChangeRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
@@ -471,9 +465,7 @@ public class TestPBImplRecords {
generateByNewInstance(ContainerLaunchContext.class);
generateByNewInstance(ApplicationSubmissionContext.class);
generateByNewInstance(ContainerReport.class);
- generateByNewInstance(ContainerResourceDecrease.class);
- generateByNewInstance(ContainerResourceIncrease.class);
- generateByNewInstance(ContainerResourceIncreaseRequest.class);
+ generateByNewInstance(ContainerResourceChangeRequest.class);
generateByNewInstance(ContainerStatus.class);
generateByNewInstance(PreemptionContainer.class);
generateByNewInstance(PreemptionResourceRequest.class);
@@ -959,21 +951,9 @@ public class TestPBImplRecords {
}
@Test
- public void testContainerResourceDecreasePBImpl() throws Exception {
- validatePBImplRecord(ContainerResourceDecreasePBImpl.class,
- ContainerResourceDecreaseProto.class);
- }
-
- @Test
- public void testContainerResourceIncreasePBImpl() throws Exception {
- validatePBImplRecord(ContainerResourceIncreasePBImpl.class,
- ContainerResourceIncreaseProto.class);
- }
-
- @Test
- public void testContainerResourceIncreaseRequestPBImpl() throws Exception {
- validatePBImplRecord(ContainerResourceIncreaseRequestPBImpl.class,
- ContainerResourceIncreaseRequestProto.class);
+ public void testContainerResourceChangeRequestPBImpl() throws Exception {
+ validatePBImplRecord(ContainerResourceChangeRequestPBImpl.class,
+ ContainerResourceChangeRequestProto.class);
}
@Test
[23/32] hadoop git commit: YARN-3867. ContainerImpl changes to
support container resizing. Contributed by Meng Ding
Posted by wa...@apache.org.
YARN-3867. ContainerImpl changes to support container resizing. Contributed by Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4173608f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4173608f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4173608f
Branch: refs/heads/YARN-1197
Commit: 4173608f428f07b1b33965ac079a9f542ddefba3
Parents: f13087f
Author: Jian He <ji...@apache.org>
Authored: Tue Jul 28 13:51:23 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:28 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/server/utils/BuilderUtils.java | 4 ++-
.../containermanager/ContainerManagerImpl.java | 7 ++--
.../container/ChangeContainerResourceEvent.java | 36 -------------------
.../containermanager/container/Container.java | 2 ++
.../container/ContainerEventType.java | 4 ---
.../container/ContainerImpl.java | 16 ++++++---
.../ChangeMonitoringContainerResourceEvent.java | 37 ++++++++++++++++++++
.../monitor/ContainersMonitorEventType.java | 3 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 11 ++++++
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../metrics/TestNodeManagerMetrics.java | 18 +++++++---
.../nodemanager/webapp/MockContainer.java | 4 +++
.../yarn/server/resourcemanager/MockNM.java | 2 +-
.../server/resourcemanager/NodeManager.java | 2 +-
.../resourcemanager/TestApplicationCleanup.java | 6 ++--
.../attempt/TestRMAppAttemptTransitions.java | 21 +++++++----
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/fifo/TestFifoScheduler.java | 4 +--
.../security/TestAMRMTokens.java | 3 +-
20 files changed, 118 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8eb876d..cb3daad 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -209,6 +209,9 @@ Release 2.8.0 - UNRELEASED
YARN-1645. ContainerManager implementation to support container resizing.
(Meng Ding & Wangda Tan via jianhe)
+ YARN-3867. ContainerImpl changes to support container resizing. (Meng Ding
+ via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index a3bd6f8..475e9fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -200,13 +200,15 @@ public class BuilderUtils {
}
public static ContainerStatus newContainerStatus(ContainerId containerId,
- ContainerState containerState, String diagnostics, int exitStatus) {
+ ContainerState containerState, String diagnostics, int exitStatus,
+ Resource capability) {
ContainerStatus containerStatus = recordFactory
.newRecordInstance(ContainerStatus.class);
containerStatus.setState(containerState);
containerStatus.setContainerId(containerId);
containerStatus.setDiagnostics(diagnostics);
containerStatus.setExitStatus(exitStatus);
+ containerStatus.setCapability(capability);
return containerStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 890a4e4..4f2ccbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -115,7 +115,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ChangeContainerResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
@@ -130,6 +129,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ChangeMonitoringContainerResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
@@ -1078,8 +1078,9 @@ public class ContainerManagerImpl extends CompositeService implements
this.readLock.lock();
try {
if (!serviceStopped) {
- dispatcher.getEventHandler().handle(new ChangeContainerResourceEvent(
- containerId, targetResource));
+ getContainersMonitor().handle(
+ new ChangeMonitoringContainerResourceEvent(
+ containerId, targetResource));
} else {
throw new YarnException(
"Unable to change container resource as the NodeManager is "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java
deleted file mode 100644
index 3944a3d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-public class ChangeContainerResourceEvent extends ContainerEvent {
-
- private Resource resource;
-
- public ChangeContainerResourceEvent(ContainerId c, Resource resource) {
- super(c, ContainerEventType.CHANGE_CONTAINER_RESOURCE);
- this.resource = resource;
- }
-
- public Resource getResource() {
- return this.resource;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 56b4fdd..1d2ec56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -37,6 +37,8 @@ public interface Container extends EventHandler<ContainerEvent> {
Resource getResource();
+ void setResource(Resource targetResource);
+
ContainerTokenIdentifier getContainerTokenIdentifier();
String getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index dc712bf..5622f8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -25,10 +25,6 @@ public enum ContainerEventType {
KILL_CONTAINER,
UPDATE_DIAGNOSTICS_MSG,
CONTAINER_DONE,
- CHANGE_CONTAINER_RESOURCE,
-
- // Producer: ContainerMonitor
- CONTAINER_RESOURCE_CHANGED,
// DownloadManager
CONTAINER_INITED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 3c76596..5c61a92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -79,6 +79,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.Resources;
public class ContainerImpl implements Container {
@@ -91,7 +92,7 @@ public class ContainerImpl implements Container {
private final ContainerLaunchContext launchContext;
private final ContainerTokenIdentifier containerTokenIdentifier;
private final ContainerId containerId;
- private final Resource resource;
+ private volatile Resource resource;
private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
@@ -249,7 +250,7 @@ public class ContainerImpl implements Container {
ContainerEventType.KILL_CONTAINER, new KillTransition())
.addTransition(ContainerState.RUNNING, ContainerState.EXITED_WITH_FAILURE,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
- new KilledExternallyTransition())
+ new KilledExternallyTransition())
// From CONTAINER_EXITED_WITH_SUCCESS State
.addTransition(ContainerState.EXITED_WITH_SUCCESS, ContainerState.DONE,
@@ -424,7 +425,7 @@ public class ContainerImpl implements Container {
this.readLock.lock();
try {
return BuilderUtils.newContainerStatus(this.containerId,
- getCurrentState(), diagnostics.toString(), exitCode);
+ getCurrentState(), diagnostics.toString(), exitCode, getResource());
} finally {
this.readLock.unlock();
}
@@ -451,7 +452,14 @@ public class ContainerImpl implements Container {
@Override
public Resource getResource() {
- return this.resource;
+ return Resources.clone(this.resource);
+ }
+
+ @Override
+ public void setResource(Resource targetResource) {
+ Resource currentResource = getResource();
+ this.resource = Resources.clone(targetResource);
+ this.metrics.changeContainer(currentResource, targetResource);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ChangeMonitoringContainerResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ChangeMonitoringContainerResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ChangeMonitoringContainerResourceEvent.java
new file mode 100644
index 0000000..e0abbed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ChangeMonitoringContainerResourceEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public class ChangeMonitoringContainerResourceEvent extends ContainersMonitorEvent {
+ private final Resource resource;
+
+ public ChangeMonitoringContainerResourceEvent(ContainerId containerId,
+ Resource resource) {
+ super(containerId,
+ ContainersMonitorEventType.CHANGE_MONITORING_CONTAINER_RESOURCE);
+ this.resource = resource;
+ }
+
+ public Resource getResource() {
+ return this.resource;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEventType.java
index be99651..2b31480 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEventType.java
@@ -20,5 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
public enum ContainersMonitorEventType {
START_MONITORING_CONTAINER,
- STOP_MONITORING_CONTAINER
+ STOP_MONITORING_CONTAINER,
+ CHANGE_MONITORING_CONTAINER_RESOURCE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index 56797d1..a38d0b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -133,6 +133,17 @@ public class NodeManagerMetrics {
availableVCores.incr(res.getVirtualCores());
}
+ public void changeContainer(Resource before, Resource now) {
+ int deltaMB = now.getMemory() - before.getMemory();
+ int deltaVCores = now.getVirtualCores() - before.getVirtualCores();
+ allocatedMB = allocatedMB + deltaMB;
+ allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
+ availableMB = availableMB - deltaMB;
+ availableGB.set((int)Math.floor(availableMB/1024d));
+ allocatedVCores.incr(deltaVCores);
+ availableVCores.decr(deltaVCores);
+ }
+
public void addResource(Resource res) {
availableMB = availableMB + res.getMemory();
availableGB.incr((int)Math.floor(availableMB/1024d));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 3c0368b..70a8f55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -1662,7 +1662,7 @@ public class TestNodeStatusUpdater {
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(contaierId, containerState,
"test_containerStatus: id=" + id + ", containerState: "
- + containerState, 0);
+ + containerState, 0, Resource.newInstance(1024, 1));
return containerStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
index 4dc4648..c0210d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
@@ -38,7 +38,12 @@ public class TestNodeManagerMetrics {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(512); //512MiB
resource.setVirtualCores(2);
-
+ Resource largerResource = Records.newRecord(Resource.class);
+ largerResource.setMemory(1024);
+ largerResource.setVirtualCores(2);
+ Resource smallerResource = Records.newRecord(Resource.class);
+ smallerResource.setMemory(256);
+ smallerResource.setVirtualCores(1);
metrics.addResource(total);
@@ -65,15 +70,20 @@ public class TestNodeManagerMetrics {
metrics.initingContainer();
metrics.runningContainer();
+ // Increase resource for a container
+ metrics.changeContainer(resource, largerResource);
+ // Decrease resource for a container
+ metrics.changeContainer(resource, smallerResource);
+
Assert.assertTrue(!metrics.containerLaunchDuration.changed());
metrics.addContainerLaunchDuration(1);
Assert.assertTrue(metrics.containerLaunchDuration.changed());
// availableGB is expected to be floored,
// while allocatedGB is expected to be ceiled.
- // allocatedGB: 3.5GB allocated memory is shown as 4GB
- // availableGB: 4.5GB available memory is shown as 4GB
- checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2);
+ // allocatedGB: 3.75GB allocated memory is shown as 4GB
+ // availableGB: 4.25GB available memory is shown as 4GB
+ checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 13, 3);
}
private void checkMetrics(int launched, int completed, int failed, int killed,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index b2ccb61..394a92c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -132,6 +132,10 @@ public class MockContainer implements Container {
}
@Override
+ public void setResource(Resource targetResource) {
+ }
+
+ @Override
public ContainerTokenIdentifier getContainerTokenIdentifier() {
return this.containerTokenIdentifier;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index c917f79..4233cd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -143,7 +143,7 @@ public class MockNM {
new HashMap<ApplicationId, List<ContainerStatus>>(1);
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(attemptId, containerId), containerState,
- "Success", 0);
+ "Success", 0, BuilderUtils.newResource(memory, vCores));
ArrayList<ContainerStatus> containerStatusList =
new ArrayList<ContainerStatus>(1);
containerStatusList.add(containerStatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index 5b7735e..b4ebf92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -193,7 +193,7 @@ public class NodeManager implements ContainerManagementProtocol {
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(container.getId(),
- ContainerState.NEW, "", -1000);
+ ContainerState.NEW, "", -1000, container.getResource());
applicationContainers.add(container);
containerStatusMap.put(container, containerStatus);
Resources.subtractFrom(available, tokenId.getResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
index 6e08aeb..3fa377e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
@@ -231,7 +231,8 @@ public class TestApplicationCleanup {
ArrayList<ContainerStatus> containerStatusList =
new ArrayList<ContainerStatus>();
containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
- .getId(), ContainerState.RUNNING, "nothing", 0));
+ .getId(), ContainerState.RUNNING, "nothing", 0,
+ conts.get(0).getResource()));
containerStatuses.put(app.getApplicationId(), containerStatusList);
NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true);
@@ -244,7 +245,8 @@ public class TestApplicationCleanup {
containerStatuses.clear();
containerStatusList.clear();
containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
- .getId(), ContainerState.RUNNING, "nothing", 0));
+ .getId(), ContainerState.RUNNING, "nothing", 0,
+ conts.get(0).getResource()));
containerStatuses.put(app.getApplicationId(), containerStatusList);
resp = nm1.nodeHeartbeat(containerStatuses, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index c8b6bd0..10ec453 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -956,7 +956,8 @@ public class TestRMAppAttemptTransitions {
int exitCode = 123;
ContainerStatus cs =
BuilderUtils.newContainerStatus(amContainer.getId(),
- ContainerState.COMPLETE, containerDiagMsg, exitCode);
+ ContainerState.COMPLETE, containerDiagMsg, exitCode,
+ amContainer.getResource());
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), cs, anyNodeId));
@@ -980,7 +981,8 @@ public class TestRMAppAttemptTransitions {
String containerDiagMsg = "some error";
int exitCode = 123;
ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
- ContainerState.COMPLETE, containerDiagMsg, exitCode);
+ ContainerState.COMPLETE, containerDiagMsg, exitCode,
+ amContainer.getResource());
ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
@@ -992,7 +994,8 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.getAppAttemptState());
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
- amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0,
+ amContainer.getResource()), anyNodeId));
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -1030,7 +1033,8 @@ public class TestRMAppAttemptTransitions {
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
- amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0,
+ amContainer.getResource()), anyNodeId));
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -1207,7 +1211,8 @@ public class TestRMAppAttemptTransitions {
BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(
applicationAttempt.getAppAttemptId(), 42),
- ContainerState.COMPLETE, "", 0), anyNodeId));
+ ContainerState.COMPLETE, "", 0,
+ amContainer.getResource()), anyNodeId));
testAppAttemptFinishingState(amContainer, finalStatus, trackingUrl,
diagnostics);
}
@@ -1227,7 +1232,8 @@ public class TestRMAppAttemptTransitions {
new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(),
BuilderUtils.newContainerStatus(amContainer.getId(),
- ContainerState.COMPLETE, "", 0), anyNodeId));
+ ContainerState.COMPLETE, "", 0,
+ amContainer.getResource()), anyNodeId));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
diagnostics, 0, false);
}
@@ -1256,7 +1262,8 @@ public class TestRMAppAttemptTransitions {
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
- amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
+ amContainer.getId(), ContainerState.COMPLETE, "", 0,
+ amContainer.getResource()), anyNodeId));
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
// send attempt_saved
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index fb7fce4..88c1444 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -870,7 +870,7 @@ public class TestCapacityScheduler {
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
- c1.getId(), ContainerState.COMPLETE, "", 0);
+ c1.getId(), ContainerState.COMPLETE, "", 0, c1.getResource());
nm1.containerStatus(containerStatus);
int waitCount = 0;
while (attempt1.getJustFinishedContainers().size() < 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 5b5c5ed..1353bdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -746,7 +746,7 @@ public class TestFifoScheduler {
Assert.assertEquals(GB, c1.getResource().getMemory());
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(c1.getId(), ContainerState.COMPLETE,
- "", 0);
+ "", 0, c1.getResource());
nm1.containerStatus(containerStatus);
int waitCount = 0;
while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) {
@@ -1141,7 +1141,7 @@ public class TestFifoScheduler {
// over-commitment.
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(c1.getId(), ContainerState.COMPLETE,
- "", 0);
+ "", 0, c1.getResource());
nm1.containerStatus(containerStatus);
int waitCount = 0;
while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4173608f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
index 5dfd092..4488ad6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
@@ -171,7 +171,8 @@ public class TestAMRMTokens {
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(attempt.getMasterContainer().getId(),
ContainerState.COMPLETE,
- "AM Container Finished", 0);
+ "AM Container Finished", 0,
+ attempt.getMasterContainer().getResource());
rm.getRMContext()
.getDispatcher()
.getEventHandler()
[09/32] hadoop git commit: YARN-4113. RM should respect
retry-interval when uses RetryPolicies.RETRY_FOREVER. (Sunil G via wangda)
Posted by wa...@apache.org.
YARN-4113. RM should respect retry-interval when uses RetryPolicies.RETRY_FOREVER. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b00392dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b00392dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b00392dd
Branch: refs/heads/YARN-1197
Commit: b00392dd9cbb6778f2f3e669e96cf7133590dfe7
Parents: c9cb6a5
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Sep 21 11:04:03 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Sep 21 11:04:03 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/yarn/client/RMProxy.java | 3 ++-
.../src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java | 3 ++-
3 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00392dd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a3dfb85..3ef5b75 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -865,6 +865,9 @@ Release 2.8.0 - UNRELEASED
YARN-4167. NPE on RMActiveServices#serviceStop when store is null. (Bibin A Chundatt via rohithsharmaks)
+ YARN-4113. RM should respect retry-interval when uses RetryPolicies.RETRY_FOREVER.
+ (Sunil G via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00392dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index 28628f3..be08f2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -232,7 +232,8 @@ public class RMProxy<T> {
RetryPolicy retryPolicy = null;
if (waitForEver) {
- retryPolicy = RetryPolicies.RETRY_FOREVER;
+ retryPolicy = RetryPolicies.retryForeverWithFixedSleep(
+ rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
} else {
retryPolicy =
RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b00392dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
index de7fc7d..8c30fd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
@@ -63,7 +63,8 @@ public class ServerProxy {
RetryPolicy retryPolicy = null;
if (maxWaitTime == -1) {
// wait forever.
- retryPolicy = RetryPolicies.RETRY_FOREVER;
+ retryPolicy = RetryPolicies.retryForeverWithFixedSleep(retryIntervalMS,
+ TimeUnit.MILLISECONDS);
} else {
retryPolicy =
RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
[10/32] hadoop git commit: YARN-4188. Make
MoveApplicationAcrossQueues abstract, newInstance static
Posted by wa...@apache.org.
YARN-4188. Make MoveApplicationAcrossQueues abstract, newInstance static
Contributed by Giovanni Matteo Fumarola
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e01b0d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e01b0d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e01b0d9
Branch: refs/heads/YARN-1197
Commit: 8e01b0d97ac3d74b049a801dfa1cc6e77d8f680a
Parents: b00392d
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Sep 21 17:51:48 2015 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Mon Sep 21 17:51:48 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../protocolrecords/MoveApplicationAcrossQueuesResponse.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01b0d9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3ef5b75..0e20d9a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -868,6 +868,9 @@ Release 2.8.0 - UNRELEASED
YARN-4113. RM should respect retry-interval when uses RetryPolicies.RETRY_FOREVER.
(Sunil G via wangda)
+ YARN-4188. Make MoveApplicationAcrossQueues abstract, newInstance static.
+ (Giovanni Matteo Fumarola via cdouglas)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e01b0d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/MoveApplicationAcrossQueuesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/MoveApplicationAcrossQueuesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/MoveApplicationAcrossQueuesResponse.java
index 109e7c4..35846dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/MoveApplicationAcrossQueuesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/MoveApplicationAcrossQueuesResponse.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.yarn.api.protocolrecords;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -36,10 +37,10 @@ import org.apache.hadoop.yarn.util.Records;
*/
@Public
@Unstable
-public class MoveApplicationAcrossQueuesResponse {
+public abstract class MoveApplicationAcrossQueuesResponse {
@Private
@Unstable
- public MoveApplicationAcrossQueuesResponse newInstance() {
+ public static MoveApplicationAcrossQueuesResponse newInstance() {
MoveApplicationAcrossQueuesResponse response =
Records.newRecord(MoveApplicationAcrossQueuesResponse.class);
return response;
[25/32] hadoop git commit: YARN-1644. RM-NM protocol changes and
NodeStatusUpdater implementation to support container resizing. Contributed
by Meng Ding
Posted by wa...@apache.org.
YARN-1644. RM-NM protocol changes and NodeStatusUpdater implementation to support container resizing. Contributed by Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e4034a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e4034a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e4034a8
Branch: refs/heads/YARN-1197
Commit: 1e4034a84f668356f47f8af8abbdc6814115c17f
Parents: de9bb82
Author: Jian He <ji...@apache.org>
Authored: Thu Aug 20 21:04:14 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:28 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/client/TestResourceTrackerOnHA.java | 2 +-
.../protocolrecords/NodeHeartbeatResponse.java | 4 +
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 76 +++++-
.../yarn/server/api/records/NodeStatus.java | 15 +-
.../api/records/impl/pb/NodeStatusPBImpl.java | 75 +++++-
.../main/proto/yarn_server_common_protos.proto | 3 +-
.../yarn_server_common_service_protos.proto | 1 +
.../hadoop/yarn/TestYarnServerApiClasses.java | 39 ++-
.../hadoop/yarn/server/nodemanager/Context.java | 3 +
.../yarn/server/nodemanager/NodeManager.java | 10 +
.../nodemanager/NodeStatusUpdaterImpl.java | 57 +++-
.../containermanager/ContainerManagerImpl.java | 159 +++++++-----
.../nodemanager/TestNodeManagerResync.java | 258 +++++++++++++++++++
.../amrmproxy/BaseAMRMProxyTest.java | 5 +
.../amrmproxy/MockResourceManagerFacade.java | 6 +-
.../containermanager/TestContainerManager.java | 2 +-
17 files changed, 628 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index acff3d6..274ffc4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -215,6 +215,9 @@ Release 2.8.0 - UNRELEASED
YARN-1643. Make ContainersMonitor support changing monitoring size of an
allocated container. (Meng Ding and Wangda Tan)
+ YARN-1644. RM-NM protocol changes and NodeStatusUpdater implementation to
+ support container resizing. (Meng Ding via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
index 6cdf87f..338198b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java
@@ -68,7 +68,7 @@ public class TestResourceTrackerOnHA extends ProtocolHATestBase{
failoverThread = createAndStartFailoverThread();
NodeStatus status =
NodeStatus.newInstance(NodeId.newInstance("localhost", 0), 0, null,
- null, null, null, null);
+ null, null, null, null, null);
NodeHeartbeatRequest request2 =
NodeHeartbeatRequest.newInstance(status, null, null,null);
resourceTracker.nodeHeartbeat(request2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 1498a0c..38fbc82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -70,4 +71,7 @@ public interface NodeHeartbeatResponse {
boolean getAreNodeLabelsAcceptedByRM();
void setAreNodeLabelsAcceptedByRM(boolean areNodeLabelsAcceptedByRM);
+
+ List<Container> getContainersToDecrease();
+ void addAllContainersToDecrease(List<Container> containersToDecrease);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index e27d8ca..12c5230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -27,12 +27,15 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
@@ -58,7 +61,9 @@ public class NodeHeartbeatResponsePBImpl extends
private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null;
-
+
+ private List<Container> containersToDecrease = null;
+
public NodeHeartbeatResponsePBImpl() {
builder = NodeHeartbeatResponseProto.newBuilder();
}
@@ -96,6 +101,9 @@ public class NodeHeartbeatResponsePBImpl extends
if (this.systemCredentials != null) {
addSystemCredentialsToProto();
}
+ if (this.containersToDecrease != null) {
+ addContainersToDecreaseToProto();
+ }
}
private void addSystemCredentialsToProto() {
@@ -408,6 +416,64 @@ public class NodeHeartbeatResponsePBImpl extends
builder.addAllApplicationsToCleanup(iterable);
}
+ private void initContainersToDecrease() {
+ if (this.containersToDecrease != null) {
+ return;
+ }
+ NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<ContainerProto> list = p.getContainersToDecreaseList();
+ this.containersToDecrease = new ArrayList<>();
+
+ for (ContainerProto c : list) {
+ this.containersToDecrease.add(convertFromProtoFormat(c));
+ }
+ }
+
+ @Override
+ public List<Container> getContainersToDecrease() {
+ initContainersToDecrease();
+ return this.containersToDecrease;
+ }
+
+ @Override
+ public void addAllContainersToDecrease(
+ final List<Container> containersToDecrease) {
+ if (containersToDecrease == null) {
+ return;
+ }
+ initContainersToDecrease();
+ this.containersToDecrease.addAll(containersToDecrease);
+ }
+
+ private void addContainersToDecreaseToProto() {
+ maybeInitBuilder();
+ builder.clearContainersToDecrease();
+ if (this.containersToDecrease == null) {
+ return;
+ }
+ Iterable<ContainerProto> iterable = new
+ Iterable<ContainerProto>() {
+ @Override
+ public Iterator<ContainerProto> iterator() {
+ return new Iterator<ContainerProto>() {
+ private Iterator<Container> iter = containersToDecrease.iterator();
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+ @Override
+ public ContainerProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ builder.addAllContainersToDecrease(iterable);
+ }
@Override
public Map<ApplicationId, ByteBuffer> getSystemCredentialsForApps() {
@@ -484,6 +550,14 @@ public class NodeHeartbeatResponsePBImpl extends
return ((MasterKeyPBImpl) t).getProto();
}
+ private ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
+ return new ContainerPBImpl(p);
+ }
+
+ private ContainerProto convertToProtoFormat(Container t) {
+ return ((ContainerPBImpl) t).getProto();
+ }
+
@Override
public boolean getAreNodeLabelsAcceptedByRM() {
NodeHeartbeatResponseProtoOrBuilder p =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
index 7b8262f..2d62db5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.util.Records;
@@ -48,6 +49,7 @@ public abstract class NodeStatus {
* @param nodeHealthStatus Health status of the node.
* @param containersUtilization Utilization of the containers in this node.
* @param nodeUtilization Utilization of the node.
+ * @param increasedContainers Containers whose resource has been increased.
* @return New {@code NodeStatus} with the provided information.
*/
public static NodeStatus newInstance(NodeId nodeId, int responseId,
@@ -55,7 +57,8 @@ public abstract class NodeStatus {
List<ApplicationId> keepAliveApplications,
NodeHealthStatus nodeHealthStatus,
ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization) {
+ ResourceUtilization nodeUtilization,
+ List<Container> increasedContainers) {
NodeStatus nodeStatus = Records.newRecord(NodeStatus.class);
nodeStatus.setResponseId(responseId);
nodeStatus.setNodeId(nodeId);
@@ -64,6 +67,7 @@ public abstract class NodeStatus {
nodeStatus.setNodeHealthStatus(nodeHealthStatus);
nodeStatus.setContainersUtilization(containersUtilization);
nodeStatus.setNodeUtilization(nodeUtilization);
+ nodeStatus.setIncreasedContainers(increasedContainers);
return nodeStatus;
}
@@ -108,4 +112,13 @@ public abstract class NodeStatus {
@Unstable
public abstract void setNodeUtilization(
ResourceUtilization nodeUtilization);
+
+ @Public
+ @Unstable
+ public abstract List<Container> getIncreasedContainers();
+
+ @Private
+ @Unstable
+ public abstract void setIncreasedContainers(
+ List<Container> increasedContainers);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
index 7d4e83f..e34451d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
@@ -24,13 +24,16 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
@@ -49,7 +52,8 @@ public class NodeStatusPBImpl extends NodeStatus {
private List<ContainerStatus> containers = null;
private NodeHealthStatus nodeHealthStatus = null;
private List<ApplicationId> keepAliveApplications = null;
-
+ private List<Container> increasedContainers = null;
+
public NodeStatusPBImpl() {
builder = NodeStatusProto.newBuilder();
}
@@ -79,6 +83,9 @@ public class NodeStatusPBImpl extends NodeStatus {
if (this.keepAliveApplications != null) {
addKeepAliveApplicationsToProto();
}
+ if (this.increasedContainers != null) {
+ addIncreasedContainersToProto();
+ }
}
private synchronized void mergeLocalToProto() {
@@ -165,6 +172,37 @@ public class NodeStatusPBImpl extends NodeStatus {
builder.addAllKeepAliveApplications(iterable);
}
+ private synchronized void addIncreasedContainersToProto() {
+ maybeInitBuilder();
+ builder.clearIncreasedContainers();
+ if (increasedContainers == null) {
+ return;
+ }
+ Iterable<ContainerProto> iterable = new
+ Iterable<ContainerProto>() {
+ @Override
+ public Iterator<ContainerProto> iterator() {
+ return new Iterator<ContainerProto>() {
+ private Iterator<Container> iter =
+ increasedContainers.iterator();
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+ @Override
+ public ContainerProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ builder.addAllIncreasedContainers(iterable);
+ }
+
@Override
public int hashCode() {
return getProto().hashCode();
@@ -336,6 +374,31 @@ public class NodeStatusPBImpl extends NodeStatus {
.setNodeUtilization(convertToProtoFormat(nodeUtilization));
}
+ @Override
+ public synchronized List<Container> getIncreasedContainers() {
+ if (increasedContainers != null) {
+ return increasedContainers;
+ }
+ NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
+ List<ContainerProto> list = p.getIncreasedContainersList();
+ this.increasedContainers = new ArrayList<>();
+ for (ContainerProto c : list) {
+ this.increasedContainers.add(convertFromProtoFormat(c));
+ }
+ return this.increasedContainers;
+ }
+
+ @Override
+ public synchronized void setIncreasedContainers(
+ List<Container> increasedContainers) {
+ maybeInitBuilder();
+ if (increasedContainers == null) {
+ builder.clearIncreasedContainers();
+ return;
+ }
+ this.increasedContainers = increasedContainers;
+ }
+
private NodeIdProto convertToProtoFormat(NodeId nodeId) {
return ((NodeIdPBImpl)nodeId).getProto();
}
@@ -377,4 +440,14 @@ public class NodeStatusPBImpl extends NodeStatus {
ResourceUtilizationProto p) {
return new ResourceUtilizationPBImpl(p);
}
+
+ private ContainerPBImpl convertFromProtoFormat(
+ ContainerProto c) {
+ return new ContainerPBImpl(c);
+ }
+
+ private ContainerProto convertToProtoFormat(
+ Container c) {
+ return ((ContainerPBImpl)c).getProto();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
index 901051f..b161f5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
@@ -38,6 +38,7 @@ message NodeStatusProto {
repeated ApplicationIdProto keep_alive_applications = 5;
optional ResourceUtilizationProto containers_utilization = 6;
optional ResourceUtilizationProto node_utilization = 7;
+ repeated ContainerProto increased_containers = 8;
}
message MasterKeyProto {
@@ -60,4 +61,4 @@ message ResourceUtilizationProto {
optional int32 pmem = 1;
optional int32 vmem = 2;
optional float cpu = 3;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index c122b2a..2db8919 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -82,6 +82,7 @@ message NodeHeartbeatResponseProto {
repeated ContainerIdProto containers_to_be_removed_from_nm = 9;
repeated SystemCredentialsForAppsProto system_credentials_for_apps = 10;
optional bool areNodeLabelsAcceptedByRM = 11 [default = false];
+ repeated ContainerProto containers_to_decrease = 12;
}
message SystemCredentialsForAppsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index d9eeb9d..c9427dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -29,6 +29,7 @@ import java.util.HashSet;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -168,6 +169,20 @@ public class TestYarnServerApiClasses {
assertTrue(copy.getAreNodeLabelsAcceptedByRM());
}
+ @Test
+ public void testNodeHeartbeatResponsePBImplWithDecreasedContainers() {
+ NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
+ original.addAllContainersToDecrease(
+ Arrays.asList(getDecreasedContainer(1, 2, 2048, 2),
+ getDecreasedContainer(2, 3, 1024, 1)));
+ NodeHeartbeatResponsePBImpl copy =
+ new NodeHeartbeatResponsePBImpl(original.getProto());
+ assertEquals(1, copy.getContainersToDecrease().get(0)
+ .getId().getContainerId());
+ assertEquals(1024, copy.getContainersToDecrease().get(1)
+ .getResource().getMemory());
+ }
+
/**
* Test RegisterNodeManagerRequestPBImpl.
*/
@@ -244,6 +259,9 @@ public class TestYarnServerApiClasses {
original.setNodeHealthStatus(getNodeHealthStatus());
original.setNodeId(getNodeId());
original.setResponseId(1);
+ original.setIncreasedContainers(
+ Arrays.asList(getIncreasedContainer(1, 2, 2048, 2),
+ getIncreasedContainer(2, 3, 4096, 3)));
NodeStatusPBImpl copy = new NodeStatusPBImpl(original.getProto());
assertEquals(3L, copy.getContainersStatuses().get(1).getContainerId()
@@ -252,7 +270,10 @@ public class TestYarnServerApiClasses {
assertEquals(1000, copy.getNodeHealthStatus().getLastHealthReportTime());
assertEquals(9090, copy.getNodeId().getPort());
assertEquals(1, copy.getResponseId());
-
+ assertEquals(1, copy.getIncreasedContainers().get(0)
+ .getId().getContainerId());
+ assertEquals(4096, copy.getIncreasedContainers().get(1)
+ .getResource().getMemory());
}
@Test
@@ -347,6 +368,22 @@ public class TestYarnServerApiClasses {
return new ApplicationIdPBImpl(appId.getProto());
}
+ private Container getDecreasedContainer(int containerID,
+ int appAttemptId, int memory, int vCores) {
+ ContainerId containerId = getContainerId(containerID, appAttemptId);
+ Resource capability = Resource.newInstance(memory, vCores);
+ return Container.newInstance(
+ containerId, null, null, capability, null, null);
+ }
+
+ private Container getIncreasedContainer(int containerID,
+ int appAttemptId, int memory, int vCores) {
+ ContainerId containerId = getContainerId(containerID, appAttemptId);
+ Resource capability = Resource.newInstance(memory, vCores);
+ return Container.newInstance(
+ containerId, null, null, capability, null, null);
+ }
+
private NodeStatus getNodeStatus() {
NodeStatus status = recordFactory.newRecordInstance(NodeStatus.class);
status.setContainersStatuses(new ArrayList<ContainerStatus>());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 52d937b..9c2d1fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -62,6 +62,9 @@ public interface Context {
ConcurrentMap<ContainerId, Container> getContainers();
+ ConcurrentMap<ContainerId, org.apache.hadoop.yarn.api.records.Container>
+ getIncreasedContainers();
+
NMContainerTokenSecretManager getContainerTokenSecretManager();
NMTokenSecretManagerInNM getNMTokenSecretManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 3cf9f1a..184f489 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -439,6 +439,10 @@ public class NodeManager extends CompositeService
protected final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
+ protected final ConcurrentMap<ContainerId,
+ org.apache.hadoop.yarn.api.records.Container> increasedContainers =
+ new ConcurrentHashMap<>();
+
private final NMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInNM nmTokenSecretManager;
private ContainerManagementProtocol containerManager;
@@ -493,6 +497,12 @@ public class NodeManager extends CompositeService
}
@Override
+ public ConcurrentMap<ContainerId, org.apache.hadoop.yarn.api.records.Container>
+ getIncreasedContainers() {
+ return this.increasedContainers;
+ }
+
+ @Override
public NMContainerTokenSecretManager getContainerTokenSecretManager() {
return this.containerTokenSecretManager;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index aa51e5c..f8ce90f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -310,18 +310,28 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
@VisibleForTesting
protected void registerWithRM()
throws YarnException, IOException {
- List<NMContainerStatus> containerReports = getNMContainerStatuses();
+ RegisterNodeManagerResponse regNMResponse;
Set<NodeLabel> nodeLabels = nodeLabelsHandler.getNodeLabelsForRegistration();
- RegisterNodeManagerRequest request =
- RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource,
- nodeManagerVersionId, containerReports, getRunningApplications(),
- nodeLabels);
- if (containerReports != null) {
- LOG.info("Registering with RM using containers :" + containerReports);
+
+ // Synchronize NM-RM registration with
+ // ContainerManagerImpl#increaseContainersResource and
+ // ContainerManagerImpl#startContainers to avoid race condition
+ // during RM recovery
+ synchronized (this.context) {
+ List<NMContainerStatus> containerReports = getNMContainerStatuses();
+ RegisterNodeManagerRequest request =
+ RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource,
+ nodeManagerVersionId, containerReports, getRunningApplications(),
+ nodeLabels);
+ if (containerReports != null) {
+ LOG.info("Registering with RM using containers :" + containerReports);
+ }
+ regNMResponse =
+ resourceTracker.registerNodeManager(request);
+ // Make sure rmIdentifier is set before we release the lock
+ this.rmIdentifier = regNMResponse.getRMIdentifier();
}
- RegisterNodeManagerResponse regNMResponse =
- resourceTracker.registerNodeManager(request);
- this.rmIdentifier = regNMResponse.getRMIdentifier();
+
// if the Resource Manager instructs NM to shutdown.
if (NodeAction.SHUTDOWN.equals(regNMResponse.getNodeAction())) {
String message =
@@ -418,10 +428,12 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
List<ContainerStatus> containersStatuses = getContainerStatuses();
ResourceUtilization containersUtilization = getContainersUtilization();
ResourceUtilization nodeUtilization = getNodeUtilization();
+ List<org.apache.hadoop.yarn.api.records.Container> increasedContainers
+ = getIncreasedContainers();
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
createKeepAliveApplicationList(), nodeHealthStatus,
- containersUtilization, nodeUtilization);
+ containersUtilization, nodeUtilization, increasedContainers);
return nodeStatus;
}
@@ -448,6 +460,21 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
return nodeResourceMonitor.getUtilization();
}
+ /* Get the containers whose resource has been increased since last
+ * NM-RM heartbeat.
+ */
+ private List<org.apache.hadoop.yarn.api.records.Container>
+ getIncreasedContainers() {
+ List<org.apache.hadoop.yarn.api.records.Container>
+ increasedContainers = new ArrayList<>(
+ this.context.getIncreasedContainers().values());
+ for (org.apache.hadoop.yarn.api.records.Container
+ container : increasedContainers) {
+ this.context.getIncreasedContainers().remove(container.getId());
+ }
+ return increasedContainers;
+ }
+
// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
// recentlyStoppedContainers collections.
@@ -765,6 +792,14 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
((NMContext) context)
.setSystemCrendentialsForApps(parseCredentials(systemCredentials));
}
+
+ List<org.apache.hadoop.yarn.api.records.Container>
+ containersToDecrease = response.getContainersToDecrease();
+ if (!containersToDecrease.isEmpty()) {
+ dispatcher.getEventHandler().handle(
+ new CMgrDecreaseContainersResourceEvent(containersToDecrease)
+ );
+ }
} catch (ConnectException e) {
//catch and throw the exception if tried MAX wait time to connect RM
dispatcher.getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 4f2ccbe..868d8d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -563,8 +563,7 @@ public class ContainerManagerImpl extends CompositeService implements
List<ApplicationId> appIds =
new ArrayList<ApplicationId>(applications.keySet());
- this.handle(
- new CMgrCompletedAppsEvent(appIds,
+ this.handle(new CMgrCompletedAppsEvent(appIds,
CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
LOG.info("Waiting for Applications to be Finished");
@@ -584,8 +583,8 @@ public class ContainerManagerImpl extends CompositeService implements
if (applications.isEmpty()) {
LOG.info("All applications in FINISHED state");
} else {
- LOG.info("Done waiting for Applications to be Finished. Still alive: " +
- applications.keySet());
+ LOG.info("Done waiting for Applications to be Finished. Still alive: "
+ + applications.keySet());
}
}
@@ -759,13 +758,12 @@ public class ContainerManagerImpl extends CompositeService implements
* Start a list of containers on this NodeManager.
*/
@Override
- public StartContainersResponse
- startContainers(StartContainersRequest requests) throws YarnException,
- IOException {
+ public StartContainersResponse startContainers(
+ StartContainersRequest requests) throws YarnException, IOException {
if (blockNewContainerRequests.get()) {
throw new NMNotYetReadyException(
- "Rejecting new containers as NodeManager has not"
- + " yet connected with ResourceManager");
+ "Rejecting new containers as NodeManager has not"
+ + " yet connected with ResourceManager");
}
UserGroupInformation remoteUgi = getRemoteUgi();
NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
@@ -773,42 +771,50 @@ public class ContainerManagerImpl extends CompositeService implements
List<ContainerId> succeededContainers = new ArrayList<ContainerId>();
Map<ContainerId, SerializedException> failedContainers =
new HashMap<ContainerId, SerializedException>();
- for (StartContainerRequest request : requests.getStartContainerRequests()) {
- ContainerId containerId = null;
- try {
- if (request.getContainerToken() == null ||
- request.getContainerToken().getIdentifier() == null) {
- throw new IOException(INVALID_CONTAINERTOKEN_MSG);
- }
- ContainerTokenIdentifier containerTokenIdentifier =
- BuilderUtils.newContainerTokenIdentifier(request.getContainerToken());
- verifyAndGetContainerTokenIdentifier(request.getContainerToken(),
- containerTokenIdentifier);
- containerId = containerTokenIdentifier.getContainerID();
-
- // Initialize the AMRMProxy service instance only if the container is of
- // type AM and if the AMRMProxy service is enabled
- if (isARMRMProxyEnabled()
- && containerTokenIdentifier.getContainerType().equals(
- ContainerType.APPLICATION_MASTER)) {
- this.amrmProxyService.processApplicationStartRequest(request);
- }
+ // Synchronize with NodeStatusUpdaterImpl#registerWithRM
+ // to avoid race condition during NM-RM resync (due to RM restart) while a
+ // container is being started, in particular when the container has not yet
+ // been added to the containers map in NMContext.
+ synchronized (this.context) {
+ for (StartContainerRequest request : requests
+ .getStartContainerRequests()) {
+ ContainerId containerId = null;
+ try {
+ if (request.getContainerToken() == null
+ || request.getContainerToken().getIdentifier() == null) {
+ throw new IOException(INVALID_CONTAINERTOKEN_MSG);
+ }
- startContainerInternal(nmTokenIdentifier,
- containerTokenIdentifier, request);
- succeededContainers.add(containerId);
- } catch (YarnException e) {
- failedContainers.put(containerId, SerializedException.newInstance(e));
- } catch (InvalidToken ie) {
- failedContainers.put(containerId, SerializedException.newInstance(ie));
- throw ie;
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(request.getContainerToken());
+ verifyAndGetContainerTokenIdentifier(request.getContainerToken(),
+ containerTokenIdentifier);
+ containerId = containerTokenIdentifier.getContainerID();
+
+ // Initialize the AMRMProxy service instance only if the container is of
+ // type AM and if the AMRMProxy service is enabled
+ if (isARMRMProxyEnabled() && containerTokenIdentifier
+ .getContainerType().equals(ContainerType.APPLICATION_MASTER)) {
+ this.amrmProxyService.processApplicationStartRequest(request);
+ }
+
+ startContainerInternal(nmTokenIdentifier, containerTokenIdentifier,
+ request);
+ succeededContainers.add(containerId);
+ } catch (YarnException e) {
+ failedContainers.put(containerId, SerializedException.newInstance(e));
+ } catch (InvalidToken ie) {
+ failedContainers
+ .put(containerId, SerializedException.newInstance(ie));
+ throw ie;
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
}
+ return StartContainersResponse
+ .newInstance(getAuxServiceMetaData(), succeededContainers,
+ failedContainers);
}
-
- return StartContainersResponse.newInstance(getAuxServiceMetaData(),
- succeededContainers, failedContainers);
}
private ContainerManagerApplicationProto buildAppProto(ApplicationId appId,
@@ -959,7 +965,7 @@ public class ContainerManagerImpl extends CompositeService implements
InvalidToken {
byte[] password =
context.getContainerTokenSecretManager().retrievePassword(
- containerTokenIdentifier);
+ containerTokenIdentifier);
byte[] tokenPass = token.getPassword().array();
if (password == null || tokenPass == null
|| !Arrays.equals(password, tokenPass)) {
@@ -989,32 +995,39 @@ public class ContainerManagerImpl extends CompositeService implements
= new ArrayList<ContainerId>();
Map<ContainerId, SerializedException> failedContainers =
new HashMap<ContainerId, SerializedException>();
- // Process container resource increase requests
- for (org.apache.hadoop.yarn.api.records.Token token :
- requests.getContainersToIncrease()) {
- ContainerId containerId = null;
- try {
- if (token.getIdentifier() == null) {
- throw new IOException(INVALID_CONTAINERTOKEN_MSG);
+ // Synchronize with NodeStatusUpdaterImpl#registerWithRM
+ // to avoid race condition during NM-RM resync (due to RM restart) while a
+ // container resource is being increased in NM, in particular when the
+ // increased container has not yet been added to the increasedContainers
+ // map in NMContext.
+ synchronized (this.context) {
+ // Process container resource increase requests
+ for (org.apache.hadoop.yarn.api.records.Token token :
+ requests.getContainersToIncrease()) {
+ ContainerId containerId = null;
+ try {
+ if (token.getIdentifier() == null) {
+ throw new IOException(INVALID_CONTAINERTOKEN_MSG);
+ }
+ ContainerTokenIdentifier containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(token);
+ verifyAndGetContainerTokenIdentifier(token,
+ containerTokenIdentifier);
+ authorizeStartAndResourceIncreaseRequest(
+ nmTokenIdentifier, containerTokenIdentifier, false);
+ containerId = containerTokenIdentifier.getContainerID();
+ // Reuse the startContainer logic to update NMToken,
+ // as container resource increase request will have come with
+ // an updated NMToken.
+ updateNMTokenIdentifier(nmTokenIdentifier);
+ Resource resource = containerTokenIdentifier.getResource();
+ changeContainerResourceInternal(containerId, resource, true);
+ successfullyIncreasedContainers.add(containerId);
+ } catch (YarnException | InvalidToken e) {
+ failedContainers.put(containerId, SerializedException.newInstance(e));
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
}
- ContainerTokenIdentifier containerTokenIdentifier =
- BuilderUtils.newContainerTokenIdentifier(token);
- verifyAndGetContainerTokenIdentifier(token,
- containerTokenIdentifier);
- authorizeStartAndResourceIncreaseRequest(
- nmTokenIdentifier, containerTokenIdentifier, false);
- containerId = containerTokenIdentifier.getContainerID();
- // Reuse the startContainer logic to update NMToken,
- // as container resource increase request will have come with
- // an updated NMToken.
- updateNMTokenIdentifier(nmTokenIdentifier);
- Resource resource = containerTokenIdentifier.getResource();
- changeContainerResourceInternal(containerId, resource, true);
- successfullyIncreasedContainers.add(containerId);
- } catch (YarnException | InvalidToken e) {
- failedContainers.put(containerId, SerializedException.newInstance(e));
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
}
}
return IncreaseContainersResourceResponse.newInstance(
@@ -1075,6 +1088,16 @@ public class ContainerManagerImpl extends CompositeService implements
+ " is not smaller than the current resource "
+ currentResource.toString());
}
+ if (increase) {
+ org.apache.hadoop.yarn.api.records.Container increasedContainer =
+ org.apache.hadoop.yarn.api.records.Container.newInstance(
+ containerId, null, null, targetResource, null, null);
+ if (context.getIncreasedContainers().putIfAbsent(containerId,
+ increasedContainer) != null){
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " resource is being increased.");
+ }
+ }
this.readLock.lock();
try {
if (!serviceStopped) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index c22d475..4250ac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -18,21 +18,35 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -41,8 +55,13 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
@@ -50,6 +69,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
@@ -57,12 +78,15 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -87,7 +111,10 @@ public class TestNodeManagerResync {
private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false);
private final NodeManagerEvent resyncEvent =
new NodeManagerEvent(NodeManagerEventType.RESYNC);
+ private final long DUMMY_RM_IDENTIFIER = 1234;
+ protected static Log LOG = LogFactory
+ .getLog(TestNodeManagerResync.class);
@Before
public void setup() throws UnsupportedFileSystemException {
@@ -209,6 +236,32 @@ public class TestNodeManagerResync {
nm.stop();
}
+ @SuppressWarnings("unchecked")
+ @Test(timeout=60000)
+ public void testContainerResourceIncreaseIsSynchronizedWithRMResync()
+ throws IOException, InterruptedException, YarnException {
+ NodeManager nm = new TestNodeManager4();
+ YarnConfiguration conf = createNMConfig();
+ conf.setBoolean(
+ YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ nm.init(conf);
+ nm.start();
+ // Start a container and make sure it is in RUNNING state
+ ((TestNodeManager4)nm).startContainer();
+ // Simulate a container resource increase in a separate thread
+ ((TestNodeManager4)nm).increaseContainersResource();
+ // Simulate RM restart by sending a RESYNC event
+ LOG.info("Sending out RESYNC event");
+ nm.getNMDispatcher().getEventHandler().handle(
+ new NodeManagerEvent(NodeManagerEventType.RESYNC));
+ try {
+ syncBarrier.await();
+ } catch (BrokenBarrierException e) {
+ e.printStackTrace();
+ }
+ Assert.assertFalse(assertionFailedInThread.get());
+ nm.stop();
+ }
// This is to test when NM gets the resync response from last heart beat, it
// should be able to send the already-sent-via-last-heart-beat container
@@ -588,6 +641,211 @@ public class TestNodeManagerResync {
}
}}
+ class TestNodeManager4 extends NodeManager {
+
+ private Thread increaseContainerResourceThread = null;
+
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ return new TestNodeStatusUpdaterImpl4(context, dispatcher,
+ healthChecker, metrics);
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(Context context,
+ ContainerExecutor exec, DeletionService del,
+ NodeStatusUpdater nodeStatusUpdater,
+ ApplicationACLsManager aclsManager,
+ LocalDirsHandlerService dirsHandler) {
+ return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
+ metrics, dirsHandler){
+ @Override
+ public void
+ setBlockNewContainerRequests(boolean blockNewContainerRequests) {
+ // do nothing
+ }
+
+ @Override
+ protected void authorizeGetAndStopContainerRequest(
+ ContainerId containerId, Container container,
+ boolean stopRequest, NMTokenIdentifier identifier)
+ throws YarnException {
+ // do nothing
+ }
+ @Override
+ protected void authorizeUser(UserGroupInformation remoteUgi,
+ NMTokenIdentifier nmTokenIdentifier) {
+ // do nothing
+ }
+ @Override
+ protected void authorizeStartAndResourceIncreaseRequest(
+ NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ boolean startRequest) throws YarnException {
+ try {
+ // Sleep 2 seconds to simulate a pro-longed increase action.
+ // If during this time a RESYNC event is sent by RM, the
+ // resync action should block until the increase action is
+ // completed.
+ // See testContainerResourceIncreaseIsSynchronizedWithRMResync()
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ @Override
+ protected void updateNMTokenIdentifier(
+ NMTokenIdentifier nmTokenIdentifier)
+ throws SecretManager.InvalidToken {
+ // Do nothing
+ }
+ @Override
+ public Map<String, ByteBuffer> getAuxServiceMetaData() {
+ return new HashMap<>();
+ }
+ @Override
+ protected NMTokenIdentifier selectNMTokenIdentifier(
+ UserGroupInformation remoteUgi) {
+ return new NMTokenIdentifier();
+ }
+ };
+ }
+
+ // Start a container in NM
+ public void startContainer()
+ throws IOException, InterruptedException, YarnException {
+ LOG.info("Start a container and wait until it is in RUNNING state");
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
+ PrintWriter fileWriter = new PrintWriter(scriptFile);
+ if (Shell.WINDOWS) {
+ fileWriter.println("@ping -n 100 127.0.0.1 >nul");
+ } else {
+ fileWriter.write("\numask 0");
+ fileWriter.write("\nexec sleep 100");
+ }
+ fileWriter.close();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ URL resource_alpha =
+ ConverterUtils.getYarnUrlFromPath(localFS
+ .makeQualified(new Path(scriptFile.getAbsolutePath())));
+ LocalResource rsrc_alpha =
+ recordFactory.newRecordInstance(LocalResource.class);
+ rsrc_alpha.setResource(resource_alpha);
+ rsrc_alpha.setSize(-1);
+ rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+ rsrc_alpha.setType(LocalResourceType.FILE);
+ rsrc_alpha.setTimestamp(scriptFile.lastModified());
+ String destinationFile = "dest_file";
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, rsrc_alpha);
+ containerLaunchContext.setLocalResources(localResources);
+ List<String> commands =
+ Arrays.asList(Shell.getRunScriptCommand(scriptFile));
+ containerLaunchContext.setCommands(commands);
+ Resource resource = Resource.newInstance(1024, 1);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(
+ containerLaunchContext,
+ getContainerToken(resource));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ getContainerManager().startContainers(allRequests);
+ // Make sure the container reaches RUNNING state
+ ContainerId cId = TestContainerManager.createContainerId(0);
+ BaseContainerManagerTest.waitForNMContainerState(
+ getContainerManager(), cId,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING);
+ }
+
+ // Increase container resource in a thread
+ public void increaseContainersResource()
+ throws InterruptedException {
+ LOG.info("Increase a container resource in a separate thread");
+ increaseContainerResourceThread = new IncreaseContainersResourceThread();
+ increaseContainerResourceThread.start();
+ }
+
+ class TestNodeStatusUpdaterImpl4 extends MockNodeStatusUpdater {
+
+ public TestNodeStatusUpdaterImpl4(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
+ }
+
+ @Override
+ protected void rebootNodeStatusUpdaterAndRegisterWithRM() {
+ try {
+ try {
+ // Check status before registerWithRM
+ List<ContainerId> containerIds = new ArrayList<>();
+ ContainerId cId = TestContainerManager.createContainerId(0);
+ containerIds.add(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
+ ContainerStatus containerStatus = getContainerManager()
+ .getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
+ assertEquals(Resource.newInstance(1024, 1),
+ containerStatus.getCapability());
+ // Call the actual rebootNodeStatusUpdaterAndRegisterWithRM().
+ // This function should be synchronized with
+ // increaseContainersResource().
+ super.rebootNodeStatusUpdaterAndRegisterWithRM();
+ // Check status after registerWithRM
+ containerStatus = getContainerManager()
+ .getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
+ assertEquals(Resource.newInstance(4096, 2),
+ containerStatus.getCapability());
+ } catch (AssertionError ae) {
+ ae.printStackTrace();
+ assertionFailedInThread.set(true);
+ } finally {
+ syncBarrier.await();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ class IncreaseContainersResourceThread extends Thread {
+ @Override
+ public void run() {
+ // Construct container resource increase request
+ List<Token> increaseTokens = new ArrayList<Token>();
+ // Add increase request.
+ Resource targetResource = Resource.newInstance(4096, 2);
+ try {
+ increaseTokens.add(getContainerToken(targetResource));
+ IncreaseContainersResourceRequest increaseRequest =
+ IncreaseContainersResourceRequest.newInstance(increaseTokens);
+ IncreaseContainersResourceResponse increaseResponse =
+ getContainerManager()
+ .increaseContainersResource(increaseRequest);
+ Assert.assertEquals(
+ 1, increaseResponse.getSuccessfullyIncreasedContainers()
+ .size());
+ Assert.assertTrue(increaseResponse.getFailedRequests().isEmpty());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private Token getContainerToken(Resource resource) throws IOException {
+ ContainerId cId = TestContainerManager.createContainerId(0);
+ return TestContainerManager.createContainerToken(
+ cId, DUMMY_RM_IDENTIFIER,
+ getNMContext().getNodeId(), user, resource,
+ getNMContext().getContainerTokenSecretManager(), null);
+ }
+ }
+
public static NMContainerStatus createNMContainerStatus(int id,
ContainerState containerState) {
ApplicationId applicationId = ApplicationId.newInstance(0, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 964379a..9bc23f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -620,6 +620,11 @@ public abstract class BaseAMRMProxyTest {
}
@Override
+ public ConcurrentMap<ContainerId, org.apache.hadoop.yarn.api.records.Container> getIncreasedContainers() {
+ return null;
+ }
+
+ @Override
public NMContainerTokenSecretManager getContainerTokenSecretManager() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java
index 7573a7a..f482784 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/MockResourceManagerFacade.java
@@ -93,8 +93,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
@@ -292,8 +290,8 @@ public class MockResourceManagerFacade implements
new ArrayList<ContainerStatus>(), containerList,
new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC, 1, null,
new ArrayList<NMToken>(),
- new ArrayList<ContainerResourceIncrease>(),
- new ArrayList<ContainerResourceDecrease>());
+ new ArrayList<Container>(),
+ new ArrayList<Container>());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e4034a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 2ea9146..3fb4112 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -108,7 +108,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
super.setup();
}
- private ContainerId createContainerId(int id) {
+ public static ContainerId createContainerId(int id) {
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
[21/32] hadoop git commit: YARN-1449. AM-NM protocol changes to
support container resizing. Contributed by Meng Ding & Wangda Tan)
Posted by wa...@apache.org.
YARN-1449. AM-NM protocol changes to support container resizing. Contributed by Meng Ding & Wangda Tan)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6694830e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6694830e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6694830e
Branch: refs/heads/YARN-1197
Commit: 6694830e42b45e455b537b93896f0b69ce76ff86
Parents: e9ffaf3
Author: Jian He <ji...@apache.org>
Authored: Tue Jul 14 16:06:25 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:27 2015 -0700
----------------------------------------------------------------------
.../v2/app/launcher/TestContainerLauncher.java | 11 +
.../app/launcher/TestContainerLauncherImpl.java | 9 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/api/ContainerManagementProtocol.java | 30 ++-
.../IncreaseContainersResourceRequest.java | 75 ++++++
.../IncreaseContainersResourceResponse.java | 93 +++++++
.../proto/containermanagement_protocol.proto | 1 +
...ContainerManagementProtocolPBClientImpl.java | 20 ++
...ontainerManagementProtocolPBServiceImpl.java | 22 ++
...IncreaseContainersResourceRequestPBImpl.java | 170 +++++++++++++
...ncreaseContainersResourceResponsePBImpl.java | 241 +++++++++++++++++++
.../hadoop/yarn/TestContainerLaunchRPC.java | 8 +
.../yarn/TestContainerResourceIncreaseRPC.java | 162 +++++++++++++
.../java/org/apache/hadoop/yarn/TestRPC.java | 8 +
.../hadoop/yarn/api/TestPBImplRecords.java | 20 ++
.../containermanager/ContainerManagerImpl.java | 13 +
.../server/resourcemanager/NodeManager.java | 11 +-
.../resourcemanager/TestAMAuthorization.java | 8 +
.../TestApplicationMasterLauncher.java | 9 +
19 files changed, 910 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index 41ee65d..6c3a4d6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -30,6 +30,8 @@ import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -449,5 +451,14 @@ public class TestContainerLauncher {
"Dummy function cause"));
throw new IOException(e);
}
+
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws IOException,
+ IOException {
+ Exception e = new Exception("Dummy function", new Exception(
+ "Dummy function cause"));
+ throw new IOException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index 184f1b2..610448c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher.EventType;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -454,6 +456,13 @@ public class TestContainerLauncherImpl {
}
@Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws YarnException,
+ IOException {
+ return null;
+ }
+
+ @Override
public void close() throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b9890a7..9ed0825 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -203,6 +203,9 @@ Release 2.8.0 - UNRELEASED
YARN-3866. AM-RM protocol changes to support container resizing. (Meng Ding
via jianhe)
+ YARN-1449. AM-NM protocol changes to support container resizing.
+ (Meng Ding & Wangda Tan via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
index 7aa43df..43e1d4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
@@ -22,6 +22,9 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -38,9 +41,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* <p>The protocol between an <code>ApplicationMaster</code> and a
- * <code>NodeManager</code> to start/stop containers and to get status
- * of running containers.</p>
- *
+ * <code>NodeManager</code> to start/stop and increase resource of containers
+ * and to get status of running containers.</p>
+ *
* <p>If security is enabled the <code>NodeManager</code> verifies that the
* <code>ApplicationMaster</code> has truly been allocated the container
* by the <code>ResourceManager</code> and also verifies all interactions such
@@ -170,4 +173,25 @@ public interface ContainerManagementProtocol {
GetContainerStatusesResponse getContainerStatuses(
GetContainerStatusesRequest request) throws YarnException,
IOException;
+
+ /**
+ * <p>
+ * The API used by the <code>ApplicationMaster</code> to request for
+ * resource increase of running containers on the <code>NodeManager</code>.
+ * </p>
+ *
+ * @param request
+ * request to increase resource of a list of containers
+ * @return response which includes a list of containerIds of containers
+ * whose resource has been successfully increased and a
+ * containerId-to-exception map for failed requests.
+ *
+ * @throws YarnException
+ * @throws IOException
+ */
+ @Public
+ @Unstable
+ IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws YarnException,
+ IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceRequest.java
new file mode 100644
index 0000000..1fe8e94
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceRequest.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import java.util.List;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>The request sent by <code>Application Master</code> to the
+ * <code>Node Manager</code> to change the resource quota of a container.</p>
+ *
+ * @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)
+ */
+@Public
+@Unstable
+public abstract class IncreaseContainersResourceRequest {
+ @Public
+ @Unstable
+ public static IncreaseContainersResourceRequest newInstance(
+ List<Token> containersToIncrease) {
+ IncreaseContainersResourceRequest request =
+ Records.newRecord(IncreaseContainersResourceRequest.class);
+ request.setContainersToIncrease(containersToIncrease);
+ return request;
+ }
+
+ /**
+ * Get a list of container tokens to be used for authorization during
+ * container resource increase.
+ * <p>
+ * Note: {@link NMToken} will be used for authenticating communication with
+ * {@code NodeManager}.
+ * @return the list of container tokens to be used for authorization during
+ * container resource increase.
+ * @see NMToken
+ */
+ @Public
+ @Unstable
+ public abstract List<Token> getContainersToIncrease();
+
+ /**
+ * Set container tokens to be used during container resource increase.
+ * The token is acquired from
+ * <code>AllocateResponse.getIncreasedContainers</code>.
+ * The token contains the container id and resource capability required for
+ * container resource increase.
+ * @param containersToIncrease the list of container tokens to be used
+ * for container resource increase.
+ */
+ @Public
+ @Unstable
+ public abstract void setContainersToIncrease(
+ List<Token> containersToIncrease);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java
new file mode 100644
index 0000000..aeb1e83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.SerializedException;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * The response sent by the <code>NodeManager</code> to the
+ * <code>ApplicationMaster</code> when asked to increase container resource.
+ * </p>
+ *
+ * @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)
+ */
+@Public
+@Unstable
+public abstract class IncreaseContainersResourceResponse {
+
+ @Private
+ @Unstable
+ public static IncreaseContainersResourceResponse newInstance(
+ List<ContainerId> successfullyIncreasedContainers,
+ Map<ContainerId, SerializedException> failedRequests) {
+ IncreaseContainersResourceResponse response =
+ Records.newRecord(IncreaseContainersResourceResponse.class);
+ response.setSuccessfullyIncreasedContainers(
+ successfullyIncreasedContainers);
+ response.setFailedRequests(failedRequests);
+ return response;
+ }
+
+ /**
+ * Get the list of containerIds of containers whose resource
+ * have been successfully increased.
+ *
+ * @return the list of containerIds of containers whose resource have
+ * been successfully increased.
+ */
+ @Public
+ @Unstable
+ public abstract List<ContainerId> getSuccessfullyIncreasedContainers();
+
+ /**
+ * Set the list of containerIds of containers whose resource have
+ * been successfully increased.
+ */
+ @Private
+ @Unstable
+ public abstract void setSuccessfullyIncreasedContainers(
+ List<ContainerId> succeedIncreasedContainers);
+
+ /**
+ * Get the containerId-to-exception map in which the exception indicates
+ * error from each container for failed requests.
+ */
+ @Public
+ @Unstable
+ public abstract Map<ContainerId, SerializedException> getFailedRequests();
+
+ /**
+ * Set the containerId-to-exception map in which the exception indicates
+ * error from each container for failed requests.
+ */
+ @Private
+ @Unstable
+ public abstract void setFailedRequests(
+ Map<ContainerId, SerializedException> failedRequests);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
index 7b1647b..f06f6cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
@@ -34,4 +34,5 @@ service ContainerManagementProtocolService {
rpc startContainers(StartContainersRequestProto) returns (StartContainersResponseProto);
rpc stopContainers(StopContainersRequestProto) returns (StopContainersResponseProto);
rpc getContainerStatuses(GetContainerStatusesRequestProto) returns (GetContainerStatusesResponseProto);
+ rpc increaseContainersResource(IncreaseContainersResourceRequestProto) returns (IncreaseContainersResourceResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
index 15397e3..ce18bde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
@@ -30,12 +30,16 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
@@ -48,6 +52,7 @@ import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
import com.google.protobuf.ServiceException;
@@ -128,4 +133,19 @@ public class ContainerManagementProtocolPBClientImpl implements ContainerManagem
return null;
}
}
+
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws YarnException,
+ IOException {
+ IncreaseContainersResourceRequestProto requestProto =
+ ((IncreaseContainersResourceRequestPBImpl)request).getProto();
+ try {
+ return new IncreaseContainersResourceResponsePBImpl(
+ proxy.increaseContainersResource(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
index 2d33e69..7626441 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
@@ -23,9 +23,12 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
@@ -33,6 +36,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRespons
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto;
@@ -94,4 +99,21 @@ public class ContainerManagementProtocolPBServiceImpl implements ContainerManage
throw new ServiceException(e);
}
}
+
+ @Override
+ public IncreaseContainersResourceResponseProto increaseContainersResource(
+ RpcController controller, IncreaseContainersResourceRequestProto proto)
+ throws ServiceException {
+ IncreaseContainersResourceRequestPBImpl request =
+ new IncreaseContainersResourceRequestPBImpl(proto);
+ try {
+ IncreaseContainersResourceResponse response =
+ real.increaseContainersResource(request);
+ return ((IncreaseContainersResourceResponsePBImpl)response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceRequestPBImpl.java
new file mode 100644
index 0000000..7417051
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceRequestPBImpl.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class IncreaseContainersResourceRequestPBImpl extends
+ IncreaseContainersResourceRequest {
+ IncreaseContainersResourceRequestProto proto =
+ IncreaseContainersResourceRequestProto.getDefaultInstance();
+ IncreaseContainersResourceRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private List<Token> containersToIncrease = null;
+
+ public IncreaseContainersResourceRequestPBImpl() {
+ builder = IncreaseContainersResourceRequestProto.newBuilder();
+ }
+
+ public IncreaseContainersResourceRequestPBImpl(
+ IncreaseContainersResourceRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public IncreaseContainersResourceRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.containersToIncrease != null) {
+ addIncreaseContainersToProto();
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = IncreaseContainersResourceRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public List<Token> getContainersToIncrease() {
+ if (containersToIncrease != null) {
+ return containersToIncrease;
+ }
+ IncreaseContainersResourceRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<TokenProto> list = p.getIncreaseContainersList();
+ containersToIncrease = new ArrayList<>();
+ for (TokenProto c : list) {
+ containersToIncrease.add(convertFromProtoFormat(c));
+ }
+ return containersToIncrease;
+ }
+
+ @Override
+ public void setContainersToIncrease(List<Token> containersToIncrease) {
+ maybeInitBuilder();
+ if (containersToIncrease == null) {
+ builder.clearIncreaseContainers();
+ }
+ this.containersToIncrease = containersToIncrease;
+ }
+
+ private void addIncreaseContainersToProto() {
+ maybeInitBuilder();
+ builder.clearIncreaseContainers();
+ if (this.containersToIncrease == null) {
+ return;
+ }
+ Iterable<TokenProto> iterable = new Iterable<TokenProto>() {
+ @Override
+ public Iterator<TokenProto> iterator() {
+ return new Iterator<TokenProto>() {
+ Iterator<Token> iter = containersToIncrease.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public TokenProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ builder.addAllIncreaseContainers(iterable);
+ }
+
+ private Token convertFromProtoFormat(TokenProto p) {
+ return new TokenPBImpl(p);
+ }
+
+ private TokenProto convertToProtoFormat(Token t) {
+ return ((TokenPBImpl) t).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceResponsePBImpl.java
new file mode 100644
index 0000000..15062e1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/IncreaseContainersResourceResponsePBImpl.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.SerializedException;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class IncreaseContainersResourceResponsePBImpl extends
+ IncreaseContainersResourceResponse {
+ IncreaseContainersResourceResponseProto proto =
+ IncreaseContainersResourceResponseProto.getDefaultInstance();
+ IncreaseContainersResourceResponseProto.Builder builder = null;
+ boolean viaProto = false;
+ private List<ContainerId> succeededRequests = null;
+ private Map<ContainerId, SerializedException> failedRequests = null;
+
+ public IncreaseContainersResourceResponsePBImpl() {
+ builder = IncreaseContainersResourceResponseProto.newBuilder();
+ }
+
+ public IncreaseContainersResourceResponsePBImpl(
+ IncreaseContainersResourceResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public IncreaseContainersResourceResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.succeededRequests != null) {
+ addSucceededRequestsToProto();
+ }
+ if (this.failedRequests != null) {
+ addFailedRequestsToProto();
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = IncreaseContainersResourceResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public List<ContainerId> getSuccessfullyIncreasedContainers() {
+ initSucceededRequests();
+ return this.succeededRequests;
+ }
+
+ @Override
+ public void setSuccessfullyIncreasedContainers(
+ List<ContainerId> succeededRequests) {
+ maybeInitBuilder();
+ if (succeededRequests == null) {
+ builder.clearSucceededRequests();
+ }
+ this.succeededRequests = succeededRequests;
+ }
+
+ private void initSucceededRequests() {
+ if (this.succeededRequests != null) {
+ return;
+ }
+ IncreaseContainersResourceResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<ContainerIdProto> list = p.getSucceededRequestsList();
+ this.succeededRequests = new ArrayList<ContainerId>();
+ for (ContainerIdProto c : list) {
+ this.succeededRequests.add(convertFromProtoFormat(c));
+ }
+ }
+
+ private void addSucceededRequestsToProto() {
+ maybeInitBuilder();
+ builder.clearSucceededRequests();
+ if (this.succeededRequests == null) {
+ return;
+ }
+ Iterable<ContainerIdProto> iterable = new Iterable<ContainerIdProto>() {
+ @Override
+ public Iterator<ContainerIdProto> iterator() {
+ return new Iterator<ContainerIdProto>() {
+ Iterator<ContainerId> iter = succeededRequests.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ContainerIdProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ builder.addAllSucceededRequests(iterable);
+ }
+
+ @Override
+ public Map<ContainerId, SerializedException> getFailedRequests() {
+ initFailedRequests();
+ return this.failedRequests;
+ }
+
+ @Override
+ public void setFailedRequests(
+ Map<ContainerId, SerializedException> failedRequests) {
+ maybeInitBuilder();
+ if (failedRequests == null) {
+ builder.clearFailedRequests();
+ }
+ this.failedRequests = failedRequests;
+ }
+
+ private void initFailedRequests() {
+ if (this.failedRequests != null) {
+ return;
+ }
+ IncreaseContainersResourceResponseProtoOrBuilder
+ p = viaProto ? proto : builder;
+ List<ContainerExceptionMapProto> protoList = p.getFailedRequestsList();
+ this.failedRequests = new HashMap<ContainerId, SerializedException>();
+ for (ContainerExceptionMapProto ce : protoList) {
+ this.failedRequests.put(convertFromProtoFormat(ce.getContainerId()),
+ convertFromProtoFormat(ce.getException()));
+ }
+ }
+
+ private void addFailedRequestsToProto() {
+ maybeInitBuilder();
+ builder.clearFailedRequests();
+ if (this.failedRequests == null) {
+ return;
+ }
+ List<ContainerExceptionMapProto> protoList =
+ new ArrayList<ContainerExceptionMapProto>();
+
+ for (Map.Entry<ContainerId, SerializedException> entry : this.failedRequests
+ .entrySet()) {
+ protoList.add(ContainerExceptionMapProto.newBuilder()
+ .setContainerId(convertToProtoFormat(entry.getKey()))
+ .setException(convertToProtoFormat(entry.getValue())).build());
+ }
+ builder.addAllFailedRequests(protoList);
+ }
+
+ private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
+ return new ContainerIdPBImpl(p);
+ }
+
+ private ContainerIdProto convertToProtoFormat(ContainerId t) {
+ return ((ContainerIdPBImpl) t).getProto();
+ }
+
+ private SerializedExceptionPBImpl convertFromProtoFormat(
+ SerializedExceptionProto p) {
+ return new SerializedExceptionPBImpl(p);
+ }
+
+ private SerializedExceptionProto convertToProtoFormat(SerializedException t) {
+ return ((SerializedExceptionPBImpl) t).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
index e2071dd..0a19783 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -166,5 +168,11 @@ public class TestContainerLaunchRPC {
GetContainerStatusesResponse.newInstance(list, null);
return null;
}
+
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws YarnException, IOException {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
new file mode 100644
index 0000000..50ff1e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.List;
+
+/*
+ * Test that the container resource increase rpc times out properly.
+ * This is used by AM to increase container resource.
+ */
+public class TestContainerResourceIncreaseRPC {
+
+ static final Log LOG = LogFactory.getLog(
+ TestContainerResourceIncreaseRPC.class);
+
+ @Test
+ public void testHadoopProtoRPCTimeout() throws Exception {
+ testRPCTimeout(HadoopYarnProtoRPC.class.getName());
+ }
+
+ private void testRPCTimeout(String rpcClass) throws Exception {
+ Configuration conf = new Configuration();
+ // set timeout low for the test
+ conf.setInt("yarn.rpc.nm-command-timeout", 3000);
+ conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
+ YarnRPC rpc = YarnRPC.create(conf);
+ String bindAddr = "localhost:0";
+ InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
+ Server server = rpc.getServer(ContainerManagementProtocol.class,
+ new DummyContainerManager(), addr, conf, null, 1);
+ server.start();
+ try {
+ ContainerManagementProtocol proxy =
+ (ContainerManagementProtocol) rpc.getProxy(
+ ContainerManagementProtocol.class,
+ server.getListenerAddress(), conf);
+ ApplicationId applicationId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId applicationAttemptId =
+ ApplicationAttemptId.newInstance(applicationId, 0);
+ ContainerId containerId =
+ ContainerId.newContainerId(applicationAttemptId, 100);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Resource resource = Resource.newInstance(1234, 2);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(containerId, "localhost", "user",
+ resource, System.currentTimeMillis() + 10000, 42, 42,
+ Priority.newInstance(0), 0);
+ Token containerToken =
+ TestRPC.newContainerToken(nodeId, "password".getBytes(),
+ containerTokenIdentifier);
+ // Construct container resource increase request,
+ List<Token> increaseTokens = new ArrayList<>();
+ increaseTokens.add(containerToken);
+ IncreaseContainersResourceRequest increaseRequest =
+ IncreaseContainersResourceRequest
+ .newInstance(increaseTokens);
+ try {
+ proxy.increaseContainersResource(increaseRequest);
+ } catch (Exception e) {
+ LOG.info(StringUtils.stringifyException(e));
+ Assert.assertEquals("Error, exception is not: "
+ + SocketTimeoutException.class.getName(),
+ SocketTimeoutException.class.getName(), e.getClass().getName());
+ return;
+ }
+ } finally {
+ server.stop();
+ }
+ Assert.fail("timeout exception should have occurred!");
+ }
+
+ public class DummyContainerManager implements ContainerManagementProtocol {
+
+ @Override
+ public StartContainersResponse startContainers(
+ StartContainersRequest requests) throws YarnException, IOException {
+ Exception e = new Exception("Dummy function", new Exception(
+ "Dummy function cause"));
+ throw new YarnException(e);
+ }
+
+ @Override
+ public StopContainersResponse
+ stopContainers(StopContainersRequest requests) throws YarnException,
+ IOException {
+ Exception e = new Exception("Dummy function", new Exception(
+ "Dummy function cause"));
+ throw new YarnException(e);
+ }
+
+ @Override
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException, IOException {
+ Exception e = new Exception("Dummy function", new Exception(
+ "Dummy function cause"));
+ throw new YarnException(e);
+ }
+
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws YarnException, IOException {
+ try {
+ // make the thread sleep to look like its not going to respond
+ Thread.sleep(10000);
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new YarnException(e);
+ }
+ throw new YarnException("Shouldn't happen!!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
index 39e6162..e718661 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
@@ -33,6 +33,8 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
@@ -219,6 +221,12 @@ public class TestRPC {
new Exception(EXCEPTION_CAUSE));
throw new YarnException(e);
}
+
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request) throws YarnException, IOException {
+ return null;
+ }
}
public static ContainerTokenIdentifier newContainerTokenIdentifier(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 0979c75..5f707b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenReque
import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl;
@@ -101,6 +103,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequest
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -278,6 +282,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionReque
import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationSubmissionResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReservationUpdateResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto;
@@ -466,6 +472,8 @@ public class TestPBImplRecords {
generateByNewInstance(ApplicationSubmissionContext.class);
generateByNewInstance(ContainerReport.class);
generateByNewInstance(ContainerResourceChangeRequest.class);
+ generateByNewInstance(IncreaseContainersResourceRequest.class);
+ generateByNewInstance(IncreaseContainersResourceResponse.class);
generateByNewInstance(ContainerStatus.class);
generateByNewInstance(PreemptionContainer.class);
generateByNewInstance(PreemptionResourceRequest.class);
@@ -871,6 +879,18 @@ public class TestPBImplRecords {
}
@Test
+ public void testIncreaseContainersResourceRequestPBImpl() throws Exception {
+ validatePBImplRecord(IncreaseContainersResourceRequestPBImpl.class,
+ IncreaseContainersResourceRequestProto.class);
+ }
+
+ @Test
+ public void testIncreaseContainersResourceResponsePBImpl() throws Exception {
+ validatePBImplRecord(IncreaseContainersResourceResponsePBImpl.class,
+ IncreaseContainersResourceResponseProto.class);
+ }
+
+ @Test
public void testSubmitApplicationRequestPBImpl() throws Exception {
validatePBImplRecord(SubmitApplicationRequestPBImpl.class,
SubmitApplicationRequestProto.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index a658e53..ba1aec2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
@@ -952,6 +954,17 @@ public class ContainerManagerImpl extends CompositeService implements
return containerTokenIdentifier;
}
+ /**
+ * Increase resource of a list of containers on this NodeManager.
+ */
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest requests)
+ throws YarnException, IOException {
+ // To be implemented in YARN-1645
+ return null;
+ }
+
@Private
@VisibleForTesting
protected void updateNMTokenIdentifier(NMTokenIdentifier nmTokenIdentifier)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index d8d474e..5b7735e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -25,6 +25,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -295,7 +297,14 @@ public class NodeManager implements ContainerManagementProtocol {
return GetContainerStatusesResponse.newInstance(statuses, null);
}
- public static org.apache.hadoop.yarn.server.api.records.NodeStatus
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request)
+ throws YarnException, IOException {
+ return null;
+ }
+
+ public static org.apache.hadoop.yarn.server.api.records.NodeStatus
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
index c7f0d0a..2787f1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
@@ -122,6 +124,12 @@ public class TestAMAuthorization {
return GetContainerStatusesResponse.newInstance(null, null);
}
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(IncreaseContainersResourceRequest request)
+ throws YarnException {
+ return IncreaseContainersResourceResponse.newInstance(null, null);
+ }
+
public Credentials getContainerCredentials() throws IOException {
Credentials credentials = new Credentials();
DataInputByteBuffer buf = new DataInputByteBuffer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6694830e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index d4f8e93..2760705 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -126,6 +128,13 @@ public class TestApplicationMasterLauncher {
GetContainerStatusesRequest request) throws YarnException {
return null;
}
+
+ @Override
+ public IncreaseContainersResourceResponse increaseContainersResource(
+ IncreaseContainersResourceRequest request)
+ throws YarnException {
+ return null;
+ }
}
@Test
[14/32] hadoop git commit: HDFS-9111. Move hdfs-client protobuf
convert methods from PBHelper to PBHelperClient. Contributed by Mingliang
Liu.
Posted by wa...@apache.org.
HDFS-9111. Move hdfs-client protobuf convert methods from PBHelper to PBHelperClient. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06022b8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06022b8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06022b8f
Branch: refs/heads/YARN-1197
Commit: 06022b8fdc40e50eaac63758246353058e8cfa6d
Parents: 8e01b0d
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Sep 21 18:53:13 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon Sep 21 18:53:13 2015 -0700
----------------------------------------------------------------------
.../hdfs/protocol/HdfsLocatedFileStatus.java | 83 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 1975 ++++++++++++++++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/protocol/HdfsLocatedFileStatus.java | 83 -
.../hdfs/protocol/datatransfer/Receiver.java | 33 +-
...tDatanodeProtocolServerSideTranslatorPB.java | 6 +-
.../protocolPB/ClientNamenodeProtocolPB.java | 4 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 128 +-
.../ClientNamenodeProtocolTranslatorPB.java | 122 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 10 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 12 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 2 +-
.../protocolPB/JournalProtocolTranslatorPB.java | 2 +-
.../NamenodeProtocolServerSideTranslatorPB.java | 3 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2038 +-----------------
.../QJournalProtocolTranslatorPB.java | 3 +-
.../hdfs/server/namenode/CacheManager.java | 4 +-
.../server/namenode/EncryptionZoneManager.java | 4 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 7 +-
.../hdfs/server/namenode/FSEditLogOp.java | 20 +-
.../server/namenode/FSImageFormatPBINode.java | 7 +-
.../snapshot/FSImageFormatPBSnapshot.java | 6 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 32 +-
24 files changed, 2289 insertions(+), 2301 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
new file mode 100644
index 0000000..23e8f57
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+
+/**
+ * Interface that represents the over the wire information
+ * including block locations for a file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class HdfsLocatedFileStatus extends HdfsFileStatus {
+ private final LocatedBlocks locations;
+
+ /**
+ * Constructor
+ *
+ * @param length size
+ * @param isdir if this is directory
+ * @param block_replication the file's replication factor
+ * @param blocksize the file's block size
+ * @param modification_time most recent modification time
+ * @param access_time most recent access time
+ * @param permission permission
+ * @param owner owner
+ * @param group group
+ * @param symlink symbolic link
+ * @param path local path name in java UTF8 format
+ * @param fileId the file id
+ * @param locations block locations
+ * @param feInfo file encryption info
+ */
+ public HdfsLocatedFileStatus(long length, boolean isdir,
+ int block_replication, long blocksize, long modification_time,
+ long access_time, FsPermission permission, String owner, String group,
+ byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
+ int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, symlink, path, fileId,
+ childrenNum, feInfo, storagePolicy);
+ this.locations = locations;
+ }
+
+ public LocatedBlocks getBlockLocations() {
+ return locations;
+ }
+
+ public final LocatedFileStatus makeQualifiedLocated(URI defaultUri,
+ Path path) {
+ return new LocatedFileStatus(getLen(), isDir(), getReplication(),
+ getBlockSize(), getModificationTime(),
+ getAccessTime(),
+ getPermission(), getOwner(), getGroup(),
+ isSymlink() ? new Path(getSymlink()) : null,
+ (getFullPath(path)).makeQualified(
+ defaultUri, null), // fully-qualify path
+ DFSUtilClient.locatedBlocks2Locations(getBlockLocations()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 98de2e9..ae0a3f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -17,46 +17,173 @@
*/
package org.apache.hadoop.hdfs.protocolPB;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.primitives.Shorts;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
import org.apache.hadoop.crypto.CipherOption;
import org.apache.hadoop.crypto.CipherSuite;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.CacheFlag;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
+import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
+import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.hdfs.protocol.CachePoolStats;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
+import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveStatsProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheFlagProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolEntryProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolStatsProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeStorageReportProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeActionProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmIdProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CryptoProtocolVersionProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DataEncryptionKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto.AdminState;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfosProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeLocalInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.StorageState;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RollingUpgradeStatusProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportEntryProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
+import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrSetFlagProto;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
import org.apache.hadoop.hdfs.util.ExactSizeInputStream;
+import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
/**
- * Utilities for converting protobuf classes to and from implementation classes
- * and other helper utilities to help in dealing with protobuf.
+ * Utilities for converting protobuf classes to and from hdfs-client side
+ * implementation classes and other helper utilities to help in dealing with
+ * protobuf.
*
* Note that when converting from an internal type to protobuf type, the
* converter never return null for protobuf type. The check for internal type
* being null must be done before calling the convert() method.
*/
public class PBHelperClient {
+ private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
+ XAttr.NameSpace.values();
+ private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES =
+ AclEntryType.values();
+ private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES =
+ AclEntryScope.values();
+ private static final FsAction[] FSACTION_VALUES =
+ FsAction.values();
+
private PBHelperClient() {
/** Hidden constructor */
}
@@ -253,7 +380,7 @@ public class PBHelperClient {
final List<StorageTypeProto> protos = new ArrayList<>(
types.length);
for (int i = startIdx; i < types.length; ++i) {
- protos.add(PBHelperClient.convertStorageType(types[i]));
+ protos.add(convertStorageType(types[i]));
}
return protos;
}
@@ -369,4 +496,1834 @@ public class PBHelperClient {
}
return null;
}
+
+ public static LocatedBlock convert(LocatedBlockProto proto) {
+ if (proto == null) return null;
+ List<DatanodeInfoProto> locs = proto.getLocsList();
+ DatanodeInfo[] targets = new DatanodeInfo[locs.size()];
+ for (int i = 0; i < locs.size(); i++) {
+ targets[i] = convert(locs.get(i));
+ }
+
+ final StorageType[] storageTypes = convertStorageTypes(
+ proto.getStorageTypesList(), locs.size());
+
+ final int storageIDsCount = proto.getStorageIDsCount();
+ final String[] storageIDs;
+ if (storageIDsCount == 0) {
+ storageIDs = null;
+ } else {
+ Preconditions.checkState(storageIDsCount == locs.size());
+ storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
+ }
+
+ // Set values from the isCached list, re-using references from loc
+ List<DatanodeInfo> cachedLocs = new ArrayList<>(locs.size());
+ List<Boolean> isCachedList = proto.getIsCachedList();
+ for (int i=0; i<isCachedList.size(); i++) {
+ if (isCachedList.get(i)) {
+ cachedLocs.add(targets[i]);
+ }
+ }
+
+ LocatedBlock lb = new LocatedBlock(convert(proto.getB()), targets,
+ storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
+ cachedLocs.toArray(new DatanodeInfo[0]));
+ lb.setBlockToken(convert(proto.getBlockToken()));
+
+ return lb;
+ }
+
+ static public DatanodeInfo convert(DatanodeInfoProto di) {
+ if (di == null) return null;
+ return new DatanodeInfo(
+ convert(di.getId()),
+ di.hasLocation() ? di.getLocation() : null,
+ di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
+ di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
+ di.getLastUpdate(), di.getLastUpdateMonotonic(),
+ di.getXceiverCount(), convert(di.getAdminState()),
+ di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
+ }
+
+ public static StorageType[] convertStorageTypes(
+ List<StorageTypeProto> storageTypesList, int expectedSize) {
+ final StorageType[] storageTypes = new StorageType[expectedSize];
+ if (storageTypesList.size() != expectedSize) {
+ // missing storage types
+ Preconditions.checkState(storageTypesList.isEmpty());
+ Arrays.fill(storageTypes, StorageType.DEFAULT);
+ } else {
+ for (int i = 0; i < storageTypes.length; ++i) {
+ storageTypes[i] = convertStorageType(storageTypesList.get(i));
+ }
+ }
+ return storageTypes;
+ }
+
+ public static Token<BlockTokenIdentifier> convert(
+ TokenProto blockToken) {
+ return new Token<>(blockToken.getIdentifier()
+ .toByteArray(), blockToken.getPassword().toByteArray(), new Text(
+ blockToken.getKind()), new Text(blockToken.getService()));
+ }
+
+ // DatanodeId
+ public static DatanodeID convert(DatanodeIDProto dn) {
+ return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
+ dn.getXferPort(), dn.getInfoPort(), dn.hasInfoSecurePort() ? dn
+ .getInfoSecurePort() : 0, dn.getIpcPort());
+ }
+
+ public static AdminStates convert(AdminState adminState) {
+ switch(adminState) {
+ case DECOMMISSION_INPROGRESS:
+ return AdminStates.DECOMMISSION_INPROGRESS;
+ case DECOMMISSIONED:
+ return AdminStates.DECOMMISSIONED;
+ case NORMAL:
+ default:
+ return AdminStates.NORMAL;
+ }
+ }
+
+ // LocatedBlocks
+ public static LocatedBlocks convert(LocatedBlocksProto lb) {
+ return new LocatedBlocks(
+ lb.getFileLength(), lb.getUnderConstruction(),
+ convertLocatedBlock(lb.getBlocksList()),
+ lb.hasLastBlock() ? convert(lb.getLastBlock()) : null,
+ lb.getIsLastBlockComplete(),
+ lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) :
+ null);
+ }
+
+ public static BlockStoragePolicy[] convertStoragePolicies(
+ List<BlockStoragePolicyProto> policyProtos) {
+ if (policyProtos == null || policyProtos.size() == 0) {
+ return new BlockStoragePolicy[0];
+ }
+ BlockStoragePolicy[] policies = new BlockStoragePolicy[policyProtos.size()];
+ int i = 0;
+ for (BlockStoragePolicyProto proto : policyProtos) {
+ policies[i++] = convert(proto);
+ }
+ return policies;
+ }
+
+ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
+ IOException {
+ final InotifyProtos.EventsListProto list = resp.getEventsList();
+ final long firstTxid = list.getFirstTxid();
+ final long lastTxid = list.getLastTxid();
+
+ List<EventBatch> batches = Lists.newArrayList();
+ if (list.getEventsList().size() > 0) {
+ throw new IOException("Can't handle old inotify server response.");
+ }
+ for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
+ long txid = bp.getTxid();
+ if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
+ throw new IOException("Error converting TxidResponseProto: got a " +
+ "transaction id " + txid + " that was outside the range of [" +
+ firstTxid + ", " + lastTxid + "].");
+ }
+ List<Event> events = Lists.newArrayList();
+ for (InotifyProtos.EventProto p : bp.getEventsList()) {
+ switch (p.getType()) {
+ case EVENT_CLOSE:
+ InotifyProtos.CloseEventProto close =
+ InotifyProtos.CloseEventProto.parseFrom(p.getContents());
+ events.add(new Event.CloseEvent(close.getPath(),
+ close.getFileSize(), close.getTimestamp()));
+ break;
+ case EVENT_CREATE:
+ InotifyProtos.CreateEventProto create =
+ InotifyProtos.CreateEventProto.parseFrom(p.getContents());
+ events.add(new Event.CreateEvent.Builder()
+ .iNodeType(createTypeConvert(create.getType()))
+ .path(create.getPath())
+ .ctime(create.getCtime())
+ .ownerName(create.getOwnerName())
+ .groupName(create.getGroupName())
+ .perms(convert(create.getPerms()))
+ .replication(create.getReplication())
+ .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
+ create.getSymlinkTarget())
+ .defaultBlockSize(create.getDefaultBlockSize())
+ .overwrite(create.getOverwrite()).build());
+ break;
+ case EVENT_METADATA:
+ InotifyProtos.MetadataUpdateEventProto meta =
+ InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
+ events.add(new Event.MetadataUpdateEvent.Builder()
+ .path(meta.getPath())
+ .metadataType(metadataUpdateTypeConvert(meta.getType()))
+ .mtime(meta.getMtime())
+ .atime(meta.getAtime())
+ .replication(meta.getReplication())
+ .ownerName(
+ meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
+ .groupName(
+ meta.getGroupName().isEmpty() ? null : meta.getGroupName())
+ .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
+ .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
+ meta.getAclsList()))
+ .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
+ meta.getXAttrsList()))
+ .xAttrsRemoved(meta.getXAttrsRemoved())
+ .build());
+ break;
+ case EVENT_RENAME:
+ InotifyProtos.RenameEventProto rename =
+ InotifyProtos.RenameEventProto.parseFrom(p.getContents());
+ events.add(new Event.RenameEvent.Builder()
+ .srcPath(rename.getSrcPath())
+ .dstPath(rename.getDestPath())
+ .timestamp(rename.getTimestamp())
+ .build());
+ break;
+ case EVENT_APPEND:
+ InotifyProtos.AppendEventProto append =
+ InotifyProtos.AppendEventProto.parseFrom(p.getContents());
+ events.add(new Event.AppendEvent.Builder().path(append.getPath())
+ .newBlock(append.hasNewBlock() && append.getNewBlock())
+ .build());
+ break;
+ case EVENT_UNLINK:
+ InotifyProtos.UnlinkEventProto unlink =
+ InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
+ events.add(new Event.UnlinkEvent.Builder()
+ .path(unlink.getPath())
+ .timestamp(unlink.getTimestamp())
+ .build());
+ break;
+ case EVENT_TRUNCATE:
+ InotifyProtos.TruncateEventProto truncate =
+ InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
+ events.add(new Event.TruncateEvent(truncate.getPath(),
+ truncate.getFileSize(), truncate.getTimestamp()));
+ break;
+ default:
+ throw new RuntimeException("Unexpected inotify event type: " +
+ p.getType());
+ }
+ }
+ batches.add(new EventBatch(txid, events.toArray(new Event[0])));
+ }
+ return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
+ resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
+ }
+
+ // Located Block Arrays and Lists
+ public static LocatedBlockProto[] convertLocatedBlock(LocatedBlock[] lb) {
+ if (lb == null) return null;
+ return convertLocatedBlock2(Arrays.asList(lb)).toArray(
+ new LocatedBlockProto[lb.length]);
+ }
+
+ public static List<LocatedBlockProto> convertLocatedBlock2(List<LocatedBlock> lb) {
+ if (lb == null) return null;
+ final int len = lb.size();
+ List<LocatedBlockProto> result = new ArrayList<>(len);
+ for (int i = 0; i < len; ++i) {
+ result.add(convert(lb.get(i)));
+ }
+ return result;
+ }
+
+ public static LocatedBlockProto convert(LocatedBlock b) {
+ if (b == null) return null;
+ Builder builder = LocatedBlockProto.newBuilder();
+ DatanodeInfo[] locs = b.getLocations();
+ List<DatanodeInfo> cachedLocs =
+ Lists.newLinkedList(Arrays.asList(b.getCachedLocations()));
+ for (int i = 0; i < locs.length; i++) {
+ DatanodeInfo loc = locs[i];
+ builder.addLocs(i, convert(loc));
+ boolean locIsCached = cachedLocs.contains(loc);
+ builder.addIsCached(locIsCached);
+ if (locIsCached) {
+ cachedLocs.remove(loc);
+ }
+ }
+ Preconditions.checkArgument(cachedLocs.size() == 0,
+ "Found additional cached replica locations that are not in the set of"
+ + " storage-backed locations!");
+
+ StorageType[] storageTypes = b.getStorageTypes();
+ if (storageTypes != null) {
+ for (int i = 0; i < storageTypes.length; ++i) {
+ builder.addStorageTypes(convertStorageType(storageTypes[i]));
+ }
+ }
+ final String[] storageIDs = b.getStorageIDs();
+ if (storageIDs != null) {
+ builder.addAllStorageIDs(Arrays.asList(storageIDs));
+ }
+
+ return builder.setB(convert(b.getBlock()))
+ .setBlockToken(convert(b.getBlockToken()))
+ .setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build();
+ }
+
+ public static BlockStoragePolicy convert(BlockStoragePolicyProto proto) {
+ List<StorageTypeProto> cList = proto.getCreationPolicy()
+ .getStorageTypesList();
+ StorageType[] creationTypes = convertStorageTypes(cList, cList.size());
+ List<StorageTypeProto> cfList = proto.hasCreationFallbackPolicy() ? proto
+ .getCreationFallbackPolicy().getStorageTypesList() : null;
+ StorageType[] creationFallbackTypes = cfList == null ? StorageType
+ .EMPTY_ARRAY : convertStorageTypes(cfList, cfList.size());
+ List<StorageTypeProto> rfList = proto.hasReplicationFallbackPolicy() ?
+ proto.getReplicationFallbackPolicy().getStorageTypesList() : null;
+ StorageType[] replicationFallbackTypes = rfList == null ? StorageType
+ .EMPTY_ARRAY : convertStorageTypes(rfList, rfList.size());
+ return new BlockStoragePolicy((byte) proto.getPolicyId(), proto.getName(),
+ creationTypes, creationFallbackTypes, replicationFallbackTypes);
+ }
+
+ public static FsActionProto convert(FsAction v) {
+ return FsActionProto.valueOf(v != null ? v.ordinal() : 0);
+ }
+
+ public static XAttrProto convertXAttrProto(XAttr a) {
+ XAttrProto.Builder builder = XAttrProto.newBuilder();
+ builder.setNamespace(convert(a.getNameSpace()));
+ if (a.getName() != null) {
+ builder.setName(a.getName());
+ }
+ if (a.getValue() != null) {
+ builder.setValue(getByteString(a.getValue()));
+ }
+ return builder.build();
+ }
+
+ public static List<XAttr> convert(ListXAttrsResponseProto a) {
+ final List<XAttrProto> xAttrs = a.getXAttrsList();
+ return convertXAttrs(xAttrs);
+ }
+
+ public static List<XAttr> convert(GetXAttrsResponseProto a) {
+ List<XAttrProto> xAttrs = a.getXAttrsList();
+ return convertXAttrs(xAttrs);
+ }
+
+ public static List<XAttr> convertXAttrs(List<XAttrProto> xAttrSpec) {
+ ArrayList<XAttr> xAttrs = Lists.newArrayListWithCapacity(xAttrSpec.size());
+ for (XAttrProto a : xAttrSpec) {
+ XAttr.Builder builder = new XAttr.Builder();
+ builder.setNameSpace(convert(a.getNamespace()));
+ if (a.hasName()) {
+ builder.setName(a.getName());
+ }
+ if (a.hasValue()) {
+ builder.setValue(a.getValue().toByteArray());
+ }
+ xAttrs.add(builder.build());
+ }
+ return xAttrs;
+ }
+
+ static XAttrNamespaceProto convert(XAttr.NameSpace v) {
+ return XAttrNamespaceProto.valueOf(v.ordinal());
+ }
+
+ static XAttr.NameSpace convert(XAttrNamespaceProto v) {
+ return castEnum(v, XATTR_NAMESPACE_VALUES);
+ }
+
+ static <T extends Enum<T>, U extends Enum<U>> U castEnum(T from, U[] to) {
+ return to[from.ordinal()];
+ }
+
+ static InotifyProtos.MetadataUpdateType metadataUpdateTypeConvert(
+ Event.MetadataUpdateEvent.MetadataType type) {
+ switch (type) {
+ case TIMES:
+ return InotifyProtos.MetadataUpdateType.META_TYPE_TIMES;
+ case REPLICATION:
+ return InotifyProtos.MetadataUpdateType.META_TYPE_REPLICATION;
+ case OWNER:
+ return InotifyProtos.MetadataUpdateType.META_TYPE_OWNER;
+ case PERMS:
+ return InotifyProtos.MetadataUpdateType.META_TYPE_PERMS;
+ case ACLS:
+ return InotifyProtos.MetadataUpdateType.META_TYPE_ACLS;
+ case XATTRS:
+ return InotifyProtos.MetadataUpdateType.META_TYPE_XATTRS;
+ default:
+ return null;
+ }
+ }
+
+ private static Event.MetadataUpdateEvent.MetadataType metadataUpdateTypeConvert(
+ InotifyProtos.MetadataUpdateType type) {
+ switch (type) {
+ case META_TYPE_TIMES:
+ return Event.MetadataUpdateEvent.MetadataType.TIMES;
+ case META_TYPE_REPLICATION:
+ return Event.MetadataUpdateEvent.MetadataType.REPLICATION;
+ case META_TYPE_OWNER:
+ return Event.MetadataUpdateEvent.MetadataType.OWNER;
+ case META_TYPE_PERMS:
+ return Event.MetadataUpdateEvent.MetadataType.PERMS;
+ case META_TYPE_ACLS:
+ return Event.MetadataUpdateEvent.MetadataType.ACLS;
+ case META_TYPE_XATTRS:
+ return Event.MetadataUpdateEvent.MetadataType.XATTRS;
+ default:
+ return null;
+ }
+ }
+
+ static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INodeType
+ type) {
+ switch (type) {
+ case DIRECTORY:
+ return InotifyProtos.INodeType.I_TYPE_DIRECTORY;
+ case FILE:
+ return InotifyProtos.INodeType.I_TYPE_FILE;
+ case SYMLINK:
+ return InotifyProtos.INodeType.I_TYPE_SYMLINK;
+ default:
+ return null;
+ }
+ }
+
+ public static List<LocatedBlock> convertLocatedBlock(
+ List<LocatedBlockProto> lb) {
+ if (lb == null) return null;
+ final int len = lb.size();
+ List<LocatedBlock> result = new ArrayList<>(len);
+ for (int i = 0; i < len; ++i) {
+ result.add(convert(lb.get(i)));
+ }
+ return result;
+ }
+
+ public static List<AclEntry> convertAclEntry(List<AclEntryProto> aclSpec) {
+ ArrayList<AclEntry> r = Lists.newArrayListWithCapacity(aclSpec.size());
+ for (AclEntryProto e : aclSpec) {
+ AclEntry.Builder builder = new AclEntry.Builder();
+ builder.setType(convert(e.getType()));
+ builder.setScope(convert(e.getScope()));
+ builder.setPermission(convert(e.getPermissions()));
+ if (e.hasName()) {
+ builder.setName(e.getName());
+ }
+ r.add(builder.build());
+ }
+ return r;
+ }
+
+ static AclEntryScopeProto convert(AclEntryScope v) {
+ return AclEntryScopeProto.valueOf(v.ordinal());
+ }
+
+ private static AclEntryScope convert(AclEntryScopeProto v) {
+ return castEnum(v, ACL_ENTRY_SCOPE_VALUES);
+ }
+
+ static AclEntryTypeProto convert(AclEntryType e) {
+ return AclEntryTypeProto.valueOf(e.ordinal());
+ }
+
+ private static AclEntryType convert(AclEntryTypeProto v) {
+ return castEnum(v, ACL_ENTRY_TYPE_VALUES);
+ }
+
+ public static FsAction convert(FsActionProto v) {
+ return castEnum(v, FSACTION_VALUES);
+ }
+
+ public static FsPermission convert(FsPermissionProto p) {
+ return new FsPermissionExtension((short)p.getPerm());
+ }
+
+ private static Event.CreateEvent.INodeType createTypeConvert(
+ InotifyProtos.INodeType type) {
+ switch (type) {
+ case I_TYPE_DIRECTORY:
+ return Event.CreateEvent.INodeType.DIRECTORY;
+ case I_TYPE_FILE:
+ return Event.CreateEvent.INodeType.FILE;
+ case I_TYPE_SYMLINK:
+ return Event.CreateEvent.INodeType.SYMLINK;
+ default:
+ return null;
+ }
+ }
+
+ public static HdfsProtos.FileEncryptionInfoProto convert(
+ FileEncryptionInfo info) {
+ if (info == null) {
+ return null;
+ }
+ return HdfsProtos.FileEncryptionInfoProto.newBuilder()
+ .setSuite(convert(info.getCipherSuite()))
+ .setCryptoProtocolVersion(convert(info.getCryptoProtocolVersion()))
+ .setKey(getByteString(info.getEncryptedDataEncryptionKey()))
+ .setIv(getByteString(info.getIV()))
+ .setEzKeyVersionName(info.getEzKeyVersionName())
+ .setKeyName(info.getKeyName())
+ .build();
+ }
+
+ public static CryptoProtocolVersionProto convert(CryptoProtocolVersion
+ version) {
+ switch(version) {
+ case UNKNOWN:
+ return CryptoProtocolVersionProto.UNKNOWN_PROTOCOL_VERSION;
+ case ENCRYPTION_ZONES:
+ return CryptoProtocolVersionProto.ENCRYPTION_ZONES;
+ default:
+ return null;
+ }
+ }
+
+ public static FileEncryptionInfo convert(
+ HdfsProtos.FileEncryptionInfoProto proto) {
+ if (proto == null) {
+ return null;
+ }
+ CipherSuite suite = convert(proto.getSuite());
+ CryptoProtocolVersion version = convert(proto.getCryptoProtocolVersion());
+ byte[] key = proto.getKey().toByteArray();
+ byte[] iv = proto.getIv().toByteArray();
+ String ezKeyVersionName = proto.getEzKeyVersionName();
+ String keyName = proto.getKeyName();
+ return new FileEncryptionInfo(suite, version, key, iv, keyName,
+ ezKeyVersionName);
+ }
+
+ public static CryptoProtocolVersion convert(CryptoProtocolVersionProto
+ proto) {
+ switch(proto) {
+ case ENCRYPTION_ZONES:
+ return CryptoProtocolVersion.ENCRYPTION_ZONES;
+ default:
+ // Set to UNKNOWN and stash the unknown enum value
+ CryptoProtocolVersion version = CryptoProtocolVersion.UNKNOWN;
+ version.setUnknownValue(proto.getNumber());
+ return version;
+ }
+ }
+
+ public static List<XAttrProto> convertXAttrProto(
+ List<XAttr> xAttrSpec) {
+ if (xAttrSpec == null) {
+ return Lists.newArrayListWithCapacity(0);
+ }
+ ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
+ xAttrSpec.size());
+ for (XAttr a : xAttrSpec) {
+ XAttrProto.Builder builder = XAttrProto.newBuilder();
+ builder.setNamespace(convert(a.getNameSpace()));
+ if (a.getName() != null) {
+ builder.setName(a.getName());
+ }
+ if (a.getValue() != null) {
+ builder.setValue(getByteString(a.getValue()));
+ }
+ xAttrs.add(builder.build());
+ }
+ return xAttrs;
+ }
+
+ /**
+ * The flag field in PB is a bitmask whose values are the same a the
+ * emum values of XAttrSetFlag
+ */
+ public static int convert(EnumSet<XAttrSetFlag> flag) {
+ int value = 0;
+ if (flag.contains(XAttrSetFlag.CREATE)) {
+ value |= XAttrSetFlagProto.XATTR_CREATE.getNumber();
+ }
+ if (flag.contains(XAttrSetFlag.REPLACE)) {
+ value |= XAttrSetFlagProto.XATTR_REPLACE.getNumber();
+ }
+ return value;
+ }
+
+ public static EncryptionZone convert(EncryptionZoneProto proto) {
+ return new EncryptionZone(proto.getId(), proto.getPath(),
+ convert(proto.getSuite()), convert(proto.getCryptoProtocolVersion()),
+ proto.getKeyName());
+ }
+
+ public static AclStatus convert(GetAclStatusResponseProto e) {
+ AclStatusProto r = e.getResult();
+ AclStatus.Builder builder = new AclStatus.Builder();
+ builder.owner(r.getOwner()).group(r.getGroup()).stickyBit(r.getSticky())
+ .addEntries(convertAclEntry(r.getEntriesList()));
+ if (r.hasPermission()) {
+ builder.setPermission(convert(r.getPermission()));
+ }
+ return builder.build();
+ }
+
+ public static List<AclEntryProto> convertAclEntryProto(
+ List<AclEntry> aclSpec) {
+ ArrayList<AclEntryProto> r = Lists.newArrayListWithCapacity(aclSpec.size());
+ for (AclEntry e : aclSpec) {
+ AclEntryProto.Builder builder = AclEntryProto.newBuilder();
+ builder.setType(convert(e.getType()));
+ builder.setScope(convert(e.getScope()));
+ builder.setPermissions(convert(e.getPermission()));
+ if (e.getName() != null) {
+ builder.setName(e.getName());
+ }
+ r.add(builder.build());
+ }
+ return r;
+ }
+
+ public static CachePoolEntry convert(CachePoolEntryProto proto) {
+ CachePoolInfo info = convert(proto.getInfo());
+ CachePoolStats stats = convert(proto.getStats());
+ return new CachePoolEntry(info, stats);
+ }
+
+ public static CachePoolInfo convert (CachePoolInfoProto proto) {
+ // Pool name is a required field, the rest are optional
+ String poolName = checkNotNull(proto.getPoolName());
+ CachePoolInfo info = new CachePoolInfo(poolName);
+ if (proto.hasOwnerName()) {
+ info.setOwnerName(proto.getOwnerName());
+ }
+ if (proto.hasGroupName()) {
+ info.setGroupName(proto.getGroupName());
+ }
+ if (proto.hasMode()) {
+ info.setMode(new FsPermission((short)proto.getMode()));
+ }
+ if (proto.hasLimit()) {
+ info.setLimit(proto.getLimit());
+ }
+ if (proto.hasMaxRelativeExpiry()) {
+ info.setMaxRelativeExpiryMs(proto.getMaxRelativeExpiry());
+ }
+ return info;
+ }
+
+ public static CachePoolStats convert (CachePoolStatsProto proto) {
+ CachePoolStats.Builder builder = new CachePoolStats.Builder();
+ builder.setBytesNeeded(proto.getBytesNeeded());
+ builder.setBytesCached(proto.getBytesCached());
+ builder.setBytesOverlimit(proto.getBytesOverlimit());
+ builder.setFilesNeeded(proto.getFilesNeeded());
+ builder.setFilesCached(proto.getFilesCached());
+ return builder.build();
+ }
+
+ public static CachePoolInfoProto convert(CachePoolInfo info) {
+ CachePoolInfoProto.Builder builder = CachePoolInfoProto.newBuilder();
+ builder.setPoolName(info.getPoolName());
+ if (info.getOwnerName() != null) {
+ builder.setOwnerName(info.getOwnerName());
+ }
+ if (info.getGroupName() != null) {
+ builder.setGroupName(info.getGroupName());
+ }
+ if (info.getMode() != null) {
+ builder.setMode(info.getMode().toShort());
+ }
+ if (info.getLimit() != null) {
+ builder.setLimit(info.getLimit());
+ }
+ if (info.getMaxRelativeExpiryMs() != null) {
+ builder.setMaxRelativeExpiry(info.getMaxRelativeExpiryMs());
+ }
+ return builder.build();
+ }
+
+ public static CacheDirectiveInfoProto convert
+ (CacheDirectiveInfo info) {
+ CacheDirectiveInfoProto.Builder builder =
+ CacheDirectiveInfoProto.newBuilder();
+ if (info.getId() != null) {
+ builder.setId(info.getId());
+ }
+ if (info.getPath() != null) {
+ builder.setPath(info.getPath().toUri().getPath());
+ }
+ if (info.getReplication() != null) {
+ builder.setReplication(info.getReplication());
+ }
+ if (info.getPool() != null) {
+ builder.setPool(info.getPool());
+ }
+ if (info.getExpiration() != null) {
+ builder.setExpiration(convert(info.getExpiration()));
+ }
+ return builder.build();
+ }
+
+ public static CacheDirectiveInfoExpirationProto convert(
+ CacheDirectiveInfo.Expiration expiration) {
+ return CacheDirectiveInfoExpirationProto.newBuilder()
+ .setIsRelative(expiration.isRelative())
+ .setMillis(expiration.getMillis())
+ .build();
+ }
+
+ public static CacheDirectiveEntry convert(CacheDirectiveEntryProto proto) {
+ CacheDirectiveInfo info = convert(proto.getInfo());
+ CacheDirectiveStats stats = convert(proto.getStats());
+ return new CacheDirectiveEntry(info, stats);
+ }
+
+ public static CacheDirectiveStats convert(CacheDirectiveStatsProto proto) {
+ CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder();
+ builder.setBytesNeeded(proto.getBytesNeeded());
+ builder.setBytesCached(proto.getBytesCached());
+ builder.setFilesNeeded(proto.getFilesNeeded());
+ builder.setFilesCached(proto.getFilesCached());
+ builder.setHasExpired(proto.getHasExpired());
+ return builder.build();
+ }
+
+ public static CacheDirectiveInfo convert
+ (CacheDirectiveInfoProto proto) {
+ CacheDirectiveInfo.Builder builder =
+ new CacheDirectiveInfo.Builder();
+ if (proto.hasId()) {
+ builder.setId(proto.getId());
+ }
+ if (proto.hasPath()) {
+ builder.setPath(new Path(proto.getPath()));
+ }
+ if (proto.hasReplication()) {
+ builder.setReplication(Shorts.checkedCast(
+ proto.getReplication()));
+ }
+ if (proto.hasPool()) {
+ builder.setPool(proto.getPool());
+ }
+ if (proto.hasExpiration()) {
+ builder.setExpiration(convert(proto.getExpiration()));
+ }
+ return builder.build();
+ }
+
+ public static CacheDirectiveInfo.Expiration convert(
+ CacheDirectiveInfoExpirationProto proto) {
+ if (proto.getIsRelative()) {
+ return CacheDirectiveInfo.Expiration.newRelative(proto.getMillis());
+ }
+ return CacheDirectiveInfo.Expiration.newAbsolute(proto.getMillis());
+ }
+
+ public static int convertCacheFlags(EnumSet<CacheFlag> flags) {
+ int value = 0;
+ if (flags.contains(CacheFlag.FORCE)) {
+ value |= CacheFlagProto.FORCE.getNumber();
+ }
+ return value;
+ }
+
+ public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) {
+ if (reportProto == null) {
+ return null;
+ }
+ String snapshotDir = reportProto.getSnapshotRoot();
+ String fromSnapshot = reportProto.getFromSnapshot();
+ String toSnapshot = reportProto.getToSnapshot();
+ List<SnapshotDiffReportEntryProto> list = reportProto
+ .getDiffReportEntriesList();
+ List<DiffReportEntry> entries = new ArrayList<>();
+ for (SnapshotDiffReportEntryProto entryProto : list) {
+ DiffReportEntry entry = convert(entryProto);
+ if (entry != null)
+ entries.add(entry);
+ }
+ return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot,
+ entries);
+ }
+
+ public static DiffReportEntry convert(SnapshotDiffReportEntryProto entry) {
+ if (entry == null) {
+ return null;
+ }
+ DiffType type = DiffType.getTypeFromLabel(entry
+ .getModificationLabel());
+ return type == null ? null : new DiffReportEntry(type, entry.getFullpath()
+ .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath()
+ .toByteArray() : null);
+ }
+
+ public static SnapshottableDirectoryStatus[] convert(
+ SnapshottableDirectoryListingProto sdlp) {
+ if (sdlp == null)
+ return null;
+ List<SnapshottableDirectoryStatusProto> list = sdlp
+ .getSnapshottableDirListingList();
+ if (list.isEmpty()) {
+ return new SnapshottableDirectoryStatus[0];
+ } else {
+ SnapshottableDirectoryStatus[] result =
+ new SnapshottableDirectoryStatus[list.size()];
+ for (int i = 0; i < list.size(); i++) {
+ result[i] = convert(list.get(i));
+ }
+ return result;
+ }
+ }
+
+ public static SnapshottableDirectoryStatus convert(
+ SnapshottableDirectoryStatusProto sdirStatusProto) {
+ if (sdirStatusProto == null) {
+ return null;
+ }
+ final HdfsFileStatusProto status = sdirStatusProto.getDirStatus();
+ return new SnapshottableDirectoryStatus(
+ status.getModificationTime(),
+ status.getAccessTime(),
+ convert(status.getPermission()),
+ status.getOwner(),
+ status.getGroup(),
+ status.getPath().toByteArray(),
+ status.getFileId(),
+ status.getChildrenNum(),
+ sdirStatusProto.getSnapshotNumber(),
+ sdirStatusProto.getSnapshotQuota(),
+ sdirStatusProto.getParentFullpath().toByteArray());
+ }
+
+ // DataEncryptionKey
+ public static DataEncryptionKey convert(DataEncryptionKeyProto bet) {
+ String encryptionAlgorithm = bet.getEncryptionAlgorithm();
+ return new DataEncryptionKey(bet.getKeyId(),
+ bet.getBlockPoolId(),
+ bet.getNonce().toByteArray(),
+ bet.getEncryptionKey().toByteArray(),
+ bet.getExpiryDate(),
+ encryptionAlgorithm.isEmpty() ? null : encryptionAlgorithm);
+ }
+
+ public static Token<DelegationTokenIdentifier> convertDelegationToken(
+ TokenProto blockToken) {
+ return new Token<>(blockToken.getIdentifier()
+ .toByteArray(), blockToken.getPassword().toByteArray(), new Text(
+ blockToken.getKind()), new Text(blockToken.getService()));
+ }
+
+ // Arrays of DatanodeId
+ public static DatanodeIDProto[] convert(DatanodeID[] did) {
+ if (did == null)
+ return null;
+ final int len = did.length;
+ DatanodeIDProto[] result = new DatanodeIDProto[len];
+ for (int i = 0; i < len; ++i) {
+ result[i] = convert(did[i]);
+ }
+ return result;
+ }
+
+ public static FsPermissionProto convert(FsPermission p) {
+ return FsPermissionProto.newBuilder().setPerm(p.toExtendedShort()).build();
+ }
+
+ public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
+ if (fs == null)
+ return null;
+ return new HdfsLocatedFileStatus(
+ fs.getLength(), fs.getFileType().equals(FileType.IS_DIR),
+ fs.getBlockReplication(), fs.getBlocksize(),
+ fs.getModificationTime(), fs.getAccessTime(),
+ convert(fs.getPermission()), fs.getOwner(), fs.getGroup(),
+ fs.getFileType().equals(FileType.IS_SYMLINK) ?
+ fs.getSymlink().toByteArray() : null,
+ fs.getPath().toByteArray(),
+ fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID,
+ fs.hasLocations() ? convert(fs.getLocations()) : null,
+ fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
+ fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null,
+ fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
+ : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
+ }
+
+ public static CorruptFileBlocks convert(CorruptFileBlocksProto c) {
+ if (c == null)
+ return null;
+ List<String> fileList = c.getFilesList();
+ return new CorruptFileBlocks(fileList.toArray(new String[fileList.size()]),
+ c.getCookie());
+ }
+
+ public static ContentSummary convert(ContentSummaryProto cs) {
+ if (cs == null) return null;
+ ContentSummary.Builder builder = new ContentSummary.Builder();
+ builder.length(cs.getLength()).
+ fileCount(cs.getFileCount()).
+ directoryCount(cs.getDirectoryCount()).
+ quota(cs.getQuota()).
+ spaceConsumed(cs.getSpaceConsumed()).
+ spaceQuota(cs.getSpaceQuota());
+ if (cs.hasTypeQuotaInfos()) {
+ for (HdfsProtos.StorageTypeQuotaInfoProto info :
+ cs.getTypeQuotaInfos().getTypeQuotaInfoList()) {
+ StorageType type = convertStorageType(info.getType());
+ builder.typeConsumed(type, info.getConsumed());
+ builder.typeQuota(type, info.getQuota());
+ }
+ }
+ return builder.build();
+ }
+
+ public static RollingUpgradeActionProto convert(RollingUpgradeAction a) {
+ switch (a) {
+ case QUERY:
+ return RollingUpgradeActionProto.QUERY;
+ case PREPARE:
+ return RollingUpgradeActionProto.START;
+ case FINALIZE:
+ return RollingUpgradeActionProto.FINALIZE;
+ default:
+ throw new IllegalArgumentException("Unexpected value: " + a);
+ }
+ }
+
+ public static RollingUpgradeInfo convert(RollingUpgradeInfoProto proto) {
+ RollingUpgradeStatusProto status = proto.getStatus();
+ return new RollingUpgradeInfo(status.getBlockPoolId(),
+ proto.getCreatedRollbackImages(),
+ proto.getStartTime(), proto.getFinalizeTime());
+ }
+
+ public static DatanodeStorageReport[] convertDatanodeStorageReports(
+ List<DatanodeStorageReportProto> protos) {
+ final DatanodeStorageReport[] reports
+ = new DatanodeStorageReport[protos.size()];
+ for(int i = 0; i < reports.length; i++) {
+ reports[i] = convertDatanodeStorageReport(protos.get(i));
+ }
+ return reports;
+ }
+
+ public static DatanodeStorageReport convertDatanodeStorageReport(
+ DatanodeStorageReportProto proto) {
+ return new DatanodeStorageReport(
+ convert(proto.getDatanodeInfo()),
+ convertStorageReports(proto.getStorageReportsList()));
+ }
+
+ public static StorageReport[] convertStorageReports(
+ List<StorageReportProto> list) {
+ final StorageReport[] report = new StorageReport[list.size()];
+ for (int i = 0; i < report.length; i++) {
+ report[i] = convert(list.get(i));
+ }
+ return report;
+ }
+
+ public static StorageReport convert(StorageReportProto p) {
+ return new StorageReport(
+ p.hasStorage() ?
+ convert(p.getStorage()) :
+ new DatanodeStorage(p.getStorageUuid()),
+ p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
+ p.getBlockPoolUsed());
+ }
+
+ public static DatanodeStorage convert(DatanodeStorageProto s) {
+ return new DatanodeStorage(s.getStorageUuid(),
+ convertState(s.getState()), convertStorageType(s.getStorageType()));
+ }
+
+ private static State convertState(StorageState state) {
+ switch(state) {
+ case READ_ONLY_SHARED:
+ return State.READ_ONLY_SHARED;
+ case NORMAL:
+ default:
+ return State.NORMAL;
+ }
+ }
+
+ public static SafeModeActionProto convert(
+ SafeModeAction a) {
+ switch (a) {
+ case SAFEMODE_LEAVE:
+ return SafeModeActionProto.SAFEMODE_LEAVE;
+ case SAFEMODE_ENTER:
+ return SafeModeActionProto.SAFEMODE_ENTER;
+ case SAFEMODE_GET:
+ return SafeModeActionProto.SAFEMODE_GET;
+ default:
+ throw new IllegalArgumentException("Unexpected SafeModeAction :" + a);
+ }
+ }
+
+ public static DatanodeInfo[] convert(List<DatanodeInfoProto> list) {
+ DatanodeInfo[] info = new DatanodeInfo[list.size()];
+ for (int i = 0; i < info.length; i++) {
+ info[i] = convert(list.get(i));
+ }
+ return info;
+ }
+
+ public static long[] convert(GetFsStatsResponseProto res) {
+ long[] result = new long[7];
+ result[ClientProtocol.GET_STATS_CAPACITY_IDX] = res.getCapacity();
+ result[ClientProtocol.GET_STATS_USED_IDX] = res.getUsed();
+ result[ClientProtocol.GET_STATS_REMAINING_IDX] = res.getRemaining();
+ result[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = res.getUnderReplicated();
+ result[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = res.getCorruptBlocks();
+ result[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = res.getMissingBlocks();
+ result[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] =
+ res.getMissingReplOneBlocks();
+ return result;
+ }
+
+ public static DatanodeReportTypeProto
+ convert(DatanodeReportType t) {
+ switch (t) {
+ case ALL: return DatanodeReportTypeProto.ALL;
+ case LIVE: return DatanodeReportTypeProto.LIVE;
+ case DEAD: return DatanodeReportTypeProto.DEAD;
+ case DECOMMISSIONING: return DatanodeReportTypeProto.DECOMMISSIONING;
+ default:
+ throw new IllegalArgumentException("Unexpected data type report:" + t);
+ }
+ }
+
+ public static DirectoryListing convert(DirectoryListingProto dl) {
+ if (dl == null)
+ return null;
+ List<HdfsFileStatusProto> partList = dl.getPartialListingList();
+ return new DirectoryListing(partList.isEmpty() ?
+ new HdfsLocatedFileStatus[0] :
+ convert(partList.toArray(new HdfsFileStatusProto[partList.size()])),
+ dl.getRemainingEntries());
+ }
+
+ public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) {
+ if (fs == null) return null;
+ final int len = fs.length;
+ HdfsFileStatus[] result = new HdfsFileStatus[len];
+ for (int i = 0; i < len; ++i) {
+ result[i] = convert(fs[i]);
+ }
+ return result;
+ }
+
+ // The creatFlag field in PB is a bitmask whose values are the same a the
+ // emum values of CreateFlag
+ public static int convertCreateFlag(EnumSetWritable<CreateFlag> flag) {
+ int value = 0;
+ if (flag.contains(CreateFlag.APPEND)) {
+ value |= CreateFlagProto.APPEND.getNumber();
+ }
+ if (flag.contains(CreateFlag.CREATE)) {
+ value |= CreateFlagProto.CREATE.getNumber();
+ }
+ if (flag.contains(CreateFlag.OVERWRITE)) {
+ value |= CreateFlagProto.OVERWRITE.getNumber();
+ }
+ if (flag.contains(CreateFlag.LAZY_PERSIST)) {
+ value |= CreateFlagProto.LAZY_PERSIST.getNumber();
+ }
+ if (flag.contains(CreateFlag.NEW_BLOCK)) {
+ value |= CreateFlagProto.NEW_BLOCK.getNumber();
+ }
+ return value;
+ }
+
+ public static FsServerDefaults convert(FsServerDefaultsProto fs) {
+ if (fs == null) return null;
+ return new FsServerDefaults(
+ fs.getBlockSize(), fs.getBytesPerChecksum(),
+ fs.getWritePacketSize(), (short) fs.getReplication(),
+ fs.getFileBufferSize(),
+ fs.getEncryptDataTransfer(),
+ fs.getTrashInterval(),
+ convert(fs.getChecksumType()));
+ }
+
+ public static List<CryptoProtocolVersionProto> convert(
+ CryptoProtocolVersion[] versions) {
+ List<CryptoProtocolVersionProto> protos =
+ Lists.newArrayListWithCapacity(versions.length);
+ for (CryptoProtocolVersion v: versions) {
+ protos.add(convert(v));
+ }
+ return protos;
+ }
+
+ static List<StorageTypesProto> convert(StorageType[][] types) {
+ List<StorageTypesProto> list = Lists.newArrayList();
+ if (types != null) {
+ for (StorageType[] ts : types) {
+ StorageTypesProto.Builder builder = StorageTypesProto.newBuilder();
+ builder.addAllStorageTypes(convertStorageTypes(ts));
+ list.add(builder.build());
+ }
+ }
+ return list;
+ }
+
+ public static BlockStoragePolicyProto convert(BlockStoragePolicy policy) {
+ BlockStoragePolicyProto.Builder builder = BlockStoragePolicyProto
+ .newBuilder().setPolicyId(policy.getId()).setName(policy.getName());
+ // creation storage types
+ StorageTypesProto creationProto = convert(policy.getStorageTypes());
+ Preconditions.checkArgument(creationProto != null);
+ builder.setCreationPolicy(creationProto);
+ // creation fallback
+ StorageTypesProto creationFallbackProto = convert(
+ policy.getCreationFallbacks());
+ if (creationFallbackProto != null) {
+ builder.setCreationFallbackPolicy(creationFallbackProto);
+ }
+ // replication fallback
+ StorageTypesProto replicationFallbackProto = convert(
+ policy.getReplicationFallbacks());
+ if (replicationFallbackProto != null) {
+ builder.setReplicationFallbackPolicy(replicationFallbackProto);
+ }
+ return builder.build();
+ }
+
+ public static StorageTypesProto convert(StorageType[] types) {
+ if (types == null || types.length == 0) {
+ return null;
+ }
+ List<StorageTypeProto> list = convertStorageTypes(types);
+ return StorageTypesProto.newBuilder().addAllStorageTypes(list).build();
+ }
+
+ public static DatanodeID[] convert(DatanodeIDProto[] did) {
+ if (did == null) return null;
+ final int len = did.length;
+ DatanodeID[] result = new DatanodeID[len];
+ for (int i = 0; i < len; ++i) {
+ result[i] = convert(did[i]);
+ }
+ return result;
+ }
+
+ // Block
+ public static BlockProto convert(Block b) {
+ return BlockProto.newBuilder().setBlockId(b.getBlockId())
+ .setGenStamp(b.getGenerationStamp()).setNumBytes(b.getNumBytes())
+ .build();
+ }
+
+ public static Block convert(BlockProto b) {
+ return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
+ }
+
+ static public DatanodeInfo[] convert(DatanodeInfoProto di[]) {
+ if (di == null) return null;
+ DatanodeInfo[] result = new DatanodeInfo[di.length];
+ for (int i = 0; i < di.length; i++) {
+ result[i] = convert(di[i]);
+ }
+ return result;
+ }
+
+ public static DatanodeStorageReportProto convertDatanodeStorageReport(
+ DatanodeStorageReport report) {
+ return DatanodeStorageReportProto.newBuilder()
+ .setDatanodeInfo(convert(report.getDatanodeInfo()))
+ .addAllStorageReports(convertStorageReports(report.getStorageReports()))
+ .build();
+ }
+
+ public static List<DatanodeStorageReportProto> convertDatanodeStorageReports(
+ DatanodeStorageReport[] reports) {
+ final List<DatanodeStorageReportProto> protos
+ = new ArrayList<>(reports.length);
+ for(int i = 0; i < reports.length; i++) {
+ protos.add(convertDatanodeStorageReport(reports[i]));
+ }
+ return protos;
+ }
+
+ public static LocatedBlock[] convertLocatedBlock(LocatedBlockProto[] lb) {
+ if (lb == null) return null;
+ return convertLocatedBlock(Arrays.asList(lb)).toArray(
+ new LocatedBlock[lb.length]);
+ }
+
+ public static LocatedBlocksProto convert(LocatedBlocks lb) {
+ if (lb == null) {
+ return null;
+ }
+ LocatedBlocksProto.Builder builder =
+ LocatedBlocksProto.newBuilder();
+ if (lb.getLastLocatedBlock() != null) {
+ builder.setLastBlock(convert(lb.getLastLocatedBlock()));
+ }
+ if (lb.getFileEncryptionInfo() != null) {
+ builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
+ }
+ return builder.setFileLength(lb.getFileLength())
+ .setUnderConstruction(lb.isUnderConstruction())
+ .addAllBlocks(convertLocatedBlock2(lb.getLocatedBlocks()))
+ .setIsLastBlockComplete(lb.isLastBlockComplete()).build();
+ }
+
+ public static DataEncryptionKeyProto convert(DataEncryptionKey bet) {
+ DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
+ .setKeyId(bet.keyId)
+ .setBlockPoolId(bet.blockPoolId)
+ .setNonce(ByteString.copyFrom(bet.nonce))
+ .setEncryptionKey(ByteString.copyFrom(bet.encryptionKey))
+ .setExpiryDate(bet.expiryDate);
+ if (bet.encryptionAlgorithm != null) {
+ b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
+ }
+ return b.build();
+ }
+
+ public static FsServerDefaultsProto convert(FsServerDefaults fs) {
+ if (fs == null) return null;
+ return FsServerDefaultsProto.newBuilder().
+ setBlockSize(fs.getBlockSize()).
+ setBytesPerChecksum(fs.getBytesPerChecksum()).
+ setWritePacketSize(fs.getWritePacketSize())
+ .setReplication(fs.getReplication())
+ .setFileBufferSize(fs.getFileBufferSize())
+ .setEncryptDataTransfer(fs.getEncryptDataTransfer())
+ .setTrashInterval(fs.getTrashInterval())
+ .setChecksumType(convert(fs.getChecksumType()))
+ .build();
+ }
+
+ public static EnumSetWritable<CreateFlag> convertCreateFlag(int flag) {
+ EnumSet<CreateFlag> result =
+ EnumSet.noneOf(CreateFlag.class);
+ if ((flag & CreateFlagProto.APPEND_VALUE) == CreateFlagProto.APPEND_VALUE) {
+ result.add(CreateFlag.APPEND);
+ }
+ if ((flag & CreateFlagProto.CREATE_VALUE) == CreateFlagProto.CREATE_VALUE) {
+ result.add(CreateFlag.CREATE);
+ }
+ if ((flag & CreateFlagProto.OVERWRITE_VALUE)
+ == CreateFlagProto.OVERWRITE_VALUE) {
+ result.add(CreateFlag.OVERWRITE);
+ }
+ if ((flag & CreateFlagProto.LAZY_PERSIST_VALUE)
+ == CreateFlagProto.LAZY_PERSIST_VALUE) {
+ result.add(CreateFlag.LAZY_PERSIST);
+ }
+ if ((flag & CreateFlagProto.NEW_BLOCK_VALUE)
+ == CreateFlagProto.NEW_BLOCK_VALUE) {
+ result.add(CreateFlag.NEW_BLOCK);
+ }
+ return new EnumSetWritable<CreateFlag>(result, CreateFlag.class);
+ }
+
+ public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
+ EnumSet<CacheFlag> result = EnumSet.noneOf(CacheFlag.class);
+ if ((flags & CacheFlagProto.FORCE_VALUE) == CacheFlagProto.FORCE_VALUE) {
+ result.add(CacheFlag.FORCE);
+ }
+ return result;
+ }
+
+ public static HdfsFileStatusProto convert(HdfsFileStatus fs) {
+ if (fs == null)
+ return null;
+ FileType fType = FileType.IS_FILE;
+ if (fs.isDir()) {
+ fType = FileType.IS_DIR;
+ } else if (fs.isSymlink()) {
+ fType = FileType.IS_SYMLINK;
+ }
+
+ HdfsFileStatusProto.Builder builder =
+ HdfsFileStatusProto.newBuilder().
+ setLength(fs.getLen()).
+ setFileType(fType).
+ setBlockReplication(fs.getReplication()).
+ setBlocksize(fs.getBlockSize()).
+ setModificationTime(fs.getModificationTime()).
+ setAccessTime(fs.getAccessTime()).
+ setPermission(convert(fs.getPermission())).
+ setOwner(fs.getOwner()).
+ setGroup(fs.getGroup()).
+ setFileId(fs.getFileId()).
+ setChildrenNum(fs.getChildrenNum()).
+ setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
+ setStoragePolicy(fs.getStoragePolicy());
+ if (fs.isSymlink()) {
+ builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
+ }
+ if (fs.getFileEncryptionInfo() != null) {
+ builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
+ }
+ if (fs instanceof HdfsLocatedFileStatus) {
+ final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
+ LocatedBlocks locations = lfs.getBlockLocations();
+ if (locations != null) {
+ builder.setLocations(convert(locations));
+ }
+ }
+ return builder.build();
+ }
+
+ public static SnapshottableDirectoryStatusProto convert(
+ SnapshottableDirectoryStatus status) {
+ if (status == null) {
+ return null;
+ }
+ int snapshotNumber = status.getSnapshotNumber();
+ int snapshotQuota = status.getSnapshotQuota();
+ byte[] parentFullPath = status.getParentFullPath();
+ ByteString parentFullPathBytes = ByteString.copyFrom(
+ parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
+ HdfsFileStatusProto fs = convert(status.getDirStatus());
+ SnapshottableDirectoryStatusProto.Builder builder =
+ SnapshottableDirectoryStatusProto
+ .newBuilder().setSnapshotNumber(snapshotNumber)
+ .setSnapshotQuota(snapshotQuota).setParentFullpath(parentFullPathBytes)
+ .setDirStatus(fs);
+ return builder.build();
+ }
+
+ public static HdfsFileStatusProto[] convert(HdfsFileStatus[] fs) {
+ if (fs == null) return null;
+ final int len = fs.length;
+ HdfsFileStatusProto[] result = new HdfsFileStatusProto[len];
+ for (int i = 0; i < len; ++i) {
+ result[i] = convert(fs[i]);
+ }
+ return result;
+ }
+
+ public static DirectoryListingProto convert(DirectoryListing d) {
+ if (d == null)
+ return null;
+ return DirectoryListingProto.newBuilder().
+ addAllPartialListing(Arrays.asList(
+ convert(d.getPartialListing()))).
+ setRemainingEntries(d.getRemainingEntries()).
+ build();
+ }
+
+ public static GetFsStatsResponseProto convert(long[] fsStats) {
+ GetFsStatsResponseProto.Builder result = GetFsStatsResponseProto
+ .newBuilder();
+ if (fsStats.length >= ClientProtocol.GET_STATS_CAPACITY_IDX + 1)
+ result.setCapacity(fsStats[ClientProtocol.GET_STATS_CAPACITY_IDX]);
+ if (fsStats.length >= ClientProtocol.GET_STATS_USED_IDX + 1)
+ result.setUsed(fsStats[ClientProtocol.GET_STATS_USED_IDX]);
+ if (fsStats.length >= ClientProtocol.GET_STATS_REMAINING_IDX + 1)
+ result.setRemaining(fsStats[ClientProtocol.GET_STATS_REMAINING_IDX]);
+ if (fsStats.length >= ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX + 1)
+ result.setUnderReplicated(
+ fsStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]);
+ if (fsStats.length >= ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX + 1)
+ result.setCorruptBlocks(
+ fsStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX]);
+ if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX + 1)
+ result.setMissingBlocks(
+ fsStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]);
+ if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX + 1)
+ result.setMissingReplOneBlocks(
+ fsStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]);
+ return result.build();
+ }
+
+ public static DatanodeReportType convert(DatanodeReportTypeProto t) {
+ switch (t) {
+ case ALL: return DatanodeReportType.ALL;
+ case LIVE: return DatanodeReportType.LIVE;
+ case DEAD: return DatanodeReportType.DEAD;
+ case DECOMMISSIONING: return DatanodeReportType.DECOMMISSIONING;
+ default:
+ throw new IllegalArgumentException("Unexpected data type report:" + t);
+ }
+ }
+
+ public static SafeModeAction convert(
+ SafeModeActionProto a) {
+ switch (a) {
+ case SAFEMODE_LEAVE:
+ return SafeModeAction.SAFEMODE_LEAVE;
+ case SAFEMODE_ENTER:
+ return SafeModeAction.SAFEMODE_ENTER;
+ case SAFEMODE_GET:
+ return SafeModeAction.SAFEMODE_GET;
+ default:
+ throw new IllegalArgumentException("Unexpected SafeModeAction :" + a);
+ }
+ }
+
+ public static RollingUpgradeAction convert(RollingUpgradeActionProto a) {
+ switch (a) {
+ case QUERY:
+ return RollingUpgradeAction.QUERY;
+ case START:
+ return RollingUpgradeAction.PREPARE;
+ case FINALIZE:
+ return RollingUpgradeAction.FINALIZE;
+ default:
+ throw new IllegalArgumentException("Unexpected value: " + a);
+ }
+ }
+
+ public static RollingUpgradeStatusProto convertRollingUpgradeStatus(
+ RollingUpgradeStatus status) {
+ return RollingUpgradeStatusProto.newBuilder()
+ .setBlockPoolId(status.getBlockPoolId())
+ .setFinalized(status.isFinalized())
+ .build();
+ }
+
+ public static RollingUpgradeStatus convert(RollingUpgradeStatusProto proto) {
+ return new RollingUpgradeStatus(proto.getBlockPoolId(),
+ proto.getFinalized());
+ }
+
+ public static RollingUpgradeInfoProto convert(RollingUpgradeInfo info) {
+ return RollingUpgradeInfoProto.newBuilder()
+ .setStatus(convertRollingUpgradeStatus(info))
+ .setCreatedRollbackImages(info.createdRollbackImages())
+ .setStartTime(info.getStartTime())
+ .setFinalizeTime(info.getFinalizeTime())
+ .build();
+ }
+
+ public static CorruptFileBlocksProto convert(CorruptFileBlocks c) {
+ if (c == null)
+ return null;
+ return CorruptFileBlocksProto.newBuilder().
+ addAllFiles(Arrays.asList(c.getFiles())).
+ setCookie(c.getCookie()).
+ build();
+ }
+
+ public static ContentSummaryProto convert(ContentSummary cs) {
+ if (cs == null) return null;
+ ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
+ builder.setLength(cs.getLength()).
+ setFileCount(cs.getFileCount()).
+ setDirectoryCount(cs.getDirectoryCount()).
+ setQuota(cs.getQuota()).
+ setSpaceConsumed(cs.getSpaceConsumed()).
+ setSpaceQuota(cs.getSpaceQuota());
+
+ if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
+ HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
+ HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
+ for (StorageType t: StorageType.getTypesSupportingQuota()) {
+ HdfsProtos.StorageTypeQuotaInfoProto info =
+ HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
+ setType(convertStorageType(t)).
+ setConsumed(cs.getTypeConsumed(t)).
+ setQuota(cs.getTypeQuota(t)).
+ build();
+ isb.addTypeQuotaInfo(info);
+ }
+ builder.setTypeQuotaInfos(isb);
+ }
+ return builder.build();
+ }
+
+ public static DatanodeStorageProto convert(DatanodeStorage s) {
+ return DatanodeStorageProto.newBuilder()
+ .setState(convertState(s.getState()))
+ .setStorageType(convertStorageType(s.getStorageType()))
+ .setStorageUuid(s.getStorageID()).build();
+ }
+
+ private static StorageState convertState(State state) {
+ switch(state) {
+ case READ_ONLY_SHARED:
+ return StorageState.READ_ONLY_SHARED;
+ case NORMAL:
+ default:
+ return StorageState.NORMAL;
+ }
+ }
+
+ public static StorageReportProto convert(StorageReport r) {
+ StorageReportProto.Builder builder = StorageReportProto.newBuilder()
+ .setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
+ .setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
+ .setStorageUuid(r.getStorage().getStorageID())
+ .setStorage(convert(r.getStorage()));
+ return builder.build();
+ }
+
+ public static List<StorageReportProto> convertStorageReports(StorageReport[] storages) {
+ final List<StorageReportProto> protos = new ArrayList<StorageReportProto>(
+ storages.length);
+ for(int i = 0; i < storages.length; i++) {
+ protos.add(convert(storages[i]));
+ }
+ return protos;
+ }
+
+ public static SnapshottableDirectoryListingProto convert(
+ SnapshottableDirectoryStatus[] status) {
+ if (status == null)
+ return null;
+ SnapshottableDirectoryStatusProto[] protos =
+ new SnapshottableDirectoryStatusProto[status.length];
+ for (int i = 0; i < status.length; i++) {
+ protos[i] = convert(status[i]);
+ }
+ List<SnapshottableDirectoryStatusProto> protoList = Arrays.asList(protos);
+ return SnapshottableDirectoryListingProto.newBuilder()
+ .addAllSnapshottableDirListing(protoList).build();
+ }
+
+ public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) {
+ if (entry == null) {
+ return null;
+ }
+ ByteString sourcePath = ByteString
+ .copyFrom(entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES : entry
+ .getSourcePath());
+ String modification = entry.getType().getLabel();
+ SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
+ .newBuilder().setFullpath(sourcePath)
+ .setModificationLabel(modification);
+ if (entry.getType() == DiffType.RENAME) {
+ ByteString targetPath = ByteString
+ .copyFrom(entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES : entry
+ .getTargetPath());
+ builder.setTargetPath(targetPath);
+ }
+ return builder.build();
+ }
+
+ public static SnapshotDiffReportProto convert(SnapshotDiffReport report) {
+ if (report == null) {
+ return null;
+ }
+ List<DiffReportEntry> entries = report.getDiffList();
+ List<SnapshotDiffReportEntryProto> entryProtos = new ArrayList<>();
+ for (DiffReportEntry entry : entries) {
+ SnapshotDiffReportEntryProto entryProto = convert(entry);
+ if (entryProto != null)
+ entryProtos.add(entryProto);
+ }
+
+ SnapshotDiffReportProto reportProto = SnapshotDiffReportProto.newBuilder()
+ .setSnapshotRoot(report.getSnapshotRoot())
+ .setFromSnapshot(report.getFromSnapshot())
+ .setToSnapshot(report.getLaterSnapshotName())
+ .addAllDiffReportEntries(entryProtos).build();
+ return reportProto;
+ }
+
+ public static CacheDirectiveStatsProto convert(CacheDirectiveStats stats) {
+ CacheDirectiveStatsProto.Builder builder =
+ CacheDirectiveStatsProto.newBuilder();
+ builder.setBytesNeeded(stats.getBytesNeeded());
+ builder.setBytesCached(stats.getBytesCached());
+ builder.setFilesNeeded(stats.getFilesNeeded());
+ builder.setFilesCached(stats.getFilesCached());
+ builder.setHasExpired(stats.hasExpired());
+ return builder.build();
+ }
+
+ public static CacheDirectiveEntryProto convert(CacheDirectiveEntry entry) {
+ CacheDirectiveEntryProto.Builder builder =
+ CacheDirectiveEntryProto.newBuilder();
+ builder.setInfo(convert(entry.getInfo()));
+ builder.setStats(convert(entry.getStats()));
+ return builder.build();
+ }
+
+ public static boolean[] convertBooleanList(
+ List<Boolean> targetPinningsList) {
+ final boolean[] targetPinnings = new boolean[targetPinningsList.size()];
+ for (int i = 0; i < targetPinningsList.size(); i++) {
+ targetPinnings[i] = targetPinningsList.get(i);
+ }
+ return targetPinnings;
+ }
+
+ public static CachePoolStatsProto convert(CachePoolStats stats) {
+ CachePoolStatsProto.Builder builder = CachePoolStatsProto.newBuilder();
+ builder.setBytesNeeded(stats.getBytesNeeded());
+ builder.setBytesCached(stats.getBytesCached());
+ builder.setBytesOverlimit(stats.getBytesOverlimit());
+ builder.setFilesNeeded(stats.getFilesNeeded());
+ builder.setFilesCached(stats.getFilesCached());
+ return builder.build();
+ }
+
+ public static CachePoolEntryProto convert(CachePoolEntry entry) {
+ CachePoolEntryProto.Builder builder = CachePoolEntryProto.newBuilder();
+ builder.setInfo(convert(entry.getInfo()));
+ builder.setStats(convert(entry.getStats()));
+ return builder.build();
+ }
+
+ public static DatanodeLocalInfoProto convert(DatanodeLocalInfo info) {
+ DatanodeLocalInfoProto.Builder builder = DatanodeLocalInfoProto.newBuilder();
+ builder.setSoftwareVersion(info.getSoftwareVersion());
+ builder.setConfigVersion(info.getConfigVersion());
+ builder.setUptime(info.getUptime());
+ return builder.build();
+ }
+
+ public static GetAclStatusResponseProto convert(AclStatus e) {
+ AclStatusProto.Builder builder = AclStatusProto.newBuilder();
+ builder.setOwner(e.getOwner())
+ .setGroup(e.getGroup()).setSticky(e.isStickyBit())
+ .addAllEntries(convertAclEntryProto(e.getEntries()));
+ if (e.getPermission() != null) {
+ builder.setPermission(convert(e.getPermission()));
+ }
+ AclStatusProto r = builder.build();
+ return GetAclStatusResponseProto.newBuilder().setResult(r).build();
+ }
+
+ public static EnumSet<XAttrSetFlag> convert(int flag) {
+ EnumSet<XAttrSetFlag> result =
+ EnumSet.noneOf(XAttrSetFlag.class);
+ if ((flag & XAttrSetFlagProto.XATTR_CREATE_VALUE) ==
+ XAttrSetFlagProto.XATTR_CREATE_VALUE) {
+ result.add(XAttrSetFlag.CREATE);
+ }
+ if ((flag & XAttrSetFlagProto.XATTR_REPLACE_VALUE) ==
+ XAttrSetFlagProto.XATTR_REPLACE_VALUE) {
+ result.add(XAttrSetFlag.REPLACE);
+ }
+ return result;
+ }
+
+ public static XAttr convertXAttr(XAttrProto a) {
+ XAttr.Builder builder = new XAttr.Builder();
+ builder.setNameSpace(convert(a.getNamespace()));
+ if (a.hasName()) {
+ builder.setName(a.getName());
+ }
+ if (a.hasValue()) {
+ builder.setValue(a.getValue().toByteArray());
+ }
+ return builder.build();
+ }
+
+ public static GetXAttrsResponseProto convertXAttrsResponse(
+ List<XAttr> xAttrs) {
+ GetXAttrsResponseProto.Builder builder = GetXAttrsResponseProto
+ .newBuilder();
+ if (xAttrs != null) {
+ builder.addAllXAttrs(convertXAttrProto(xAttrs));
+ }
+ return builder.build();
+ }
+
+ public static ListXAttrsResponseProto convertListXAttrsResponse(
+ List<XAttr> names) {
+ ListXAttrsResponseProto.Builder builder =
+ ListXAttrsResponseProto.newBuilder();
+ if (names != null) {
+ builder.addAllXAttrs(convertXAttrProto(names));
+ }
+ return builder.build();
+ }
+
+ public static EncryptionZoneProto convert(EncryptionZone zone) {
+ return EncryptionZoneProto.newBuilder()
+ .setId(zone.getId())
+ .setPath(zone.getPath())
+ .setSuite(convert(zone.getSuite()))
+ .setCryptoProtocolVersion(convert(zone.getVersion()))
+ .setKeyName(zone.getKeyName())
+ .build();
+ }
+
+ public static SlotId convert(ShortCircuitShmSlotProto slotId) {
+ return new SlotId(convert(slotId.getShmId()),
+ slotId.getSlotIdx());
+ }
+
+ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList el) {
+ InotifyProtos.EventsListProto.Builder builder =
+ InotifyProtos.EventsListProto.newBuilder();
+ for (EventBatch b : el.getBatches()) {
+ List<InotifyProtos.EventProto> events = Lists.newArrayList();
+ for (Event e : b.getEvents()) {
+ switch (e.getEventType()) {
+ case CLOSE:
+ Event.CloseEvent ce = (Event.CloseEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_CLOSE)
+ .setContents(
+ InotifyProtos.CloseEventProto.newBuilder()
+ .setPath(ce.getPath())
+ .setFileSize(ce.getFileSize())
+ .setTimestamp(ce.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case CREATE:
+ Event.CreateEvent ce2 = (Event.CreateEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_CREATE)
+ .setContents(
+ InotifyProtos.CreateEventProto.newBuilder()
+ .setType(createTypeConvert(ce2.getiNodeType()))
+ .setPath(ce2.getPath())
+ .setCtime(ce2.getCtime())
+ .setOwnerName(ce2.getOwnerName())
+ .setGroupName(ce2.getGroupName())
+ .setPerms(convert(ce2.getPerms()))
+ .setReplication(ce2.getReplication())
+ .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
+ "" : ce2.getSymlinkTarget())
+ .setDefaultBlockSize(ce2.getDefaultBlockSize())
+ .setOverwrite(ce2.getOverwrite()).build().toByteString()
+ ).build());
+ break;
+ case METADATA:
+ Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
+ InotifyProtos.MetadataUpdateEventProto.Builder metaB =
+ InotifyProtos.MetadataUpdateEventProto.newBuilder()
+ .setPath(me.getPath())
+ .setType(metadataUpdateTypeConvert(me.getMetadataType()))
+ .setMtime(me.getMtime())
+ .setAtime(me.getAtime())
+ .setReplication(me.getReplication())
+ .setOwnerName(me.getOwnerName() == null ? "" :
+ me.getOwnerName())
+ .setGroupName(me.getGroupName() == null ? "" :
+ me.getGroupName())
+ .addAllAcls(me.getAcls() == null ?
+ Lists.<AclEntryProto>newArrayList() :
+ convertAclEntryProto(me.getAcls()))
+ .addAllXAttrs(me.getxAttrs() == null ?
+ Lists.<XAttrProto>newArrayList() :
+ convertXAttrProto(me.getxAttrs()))
+ .setXAttrsRemoved(me.isxAttrsRemoved());
+ if (me.getPerms() != null) {
+ metaB.setPerms(convert(me.getPerms()));
+ }
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_METADATA)
+ .setContents(metaB.build().toByteString())
+ .build());
+ break;
+ case RENAME:
+ Event.RenameEvent re = (Event.RenameEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_RENAME)
+ .setContents(
+ InotifyProtos.RenameEventProto.newBuilder()
+ .setSrcPath(re.getSrcPath())
+ .setDestPath(re.getDstPath())
+ .setTimestamp(re.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case APPEND:
+ Event.AppendEvent re2 = (Event.AppendEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_APPEND)
+ .setContents(InotifyProtos.AppendEventProto.newBuilder()
+ .setPath(re2.getPath())
+ .setNewBlock(re2.toNewBlock()).build().toByteString())
+ .build());
+ break;
+ case UNLINK:
+ Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_UNLINK)
+ .setContents(
+ InotifyProtos.UnlinkEventProto.newBuilder()
+ .setPath(ue.getPath())
+ .setTimestamp(ue.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case TRUNCATE:
+ Event.TruncateEvent te = (Event.TruncateEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_TRUNCATE)
+ .setContents(
+ InotifyProtos.TruncateEventProto.newBuilder()
+ .setPath(te.getPath())
+ .setFileSize(te.getFileSize())
+ .setTimestamp(te.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ default:
+ throw new RuntimeException("Unexpected inotify event: " + e);
+ }
+ }
+ builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().
+ setTxid(b.getTxid()).
+ addAllEvents(events));
+ }
+ builder.setFirstTxid(el.getFirstTxid());
+ builder.setLastTxid(el.getLastTxid());
+ builder.setSyncTxid(el.getSyncTxid());
+ return GetEditsFromTxidResponseProto.newBuilder().setEventsList(
+ builder.build()).build();
+ }
+
+ public static CryptoProtocolVersion[] convertCryptoProtocolVersions(
+ List<CryptoProtocolVersionProto> protos) {
+ List<CryptoProtocolVersion> versions =
+ Lists.newArrayListWithCapacity(protos.size());
+ for (CryptoProtocolVersionProto p: protos) {
+ versions.add(convert(p));
+ }
+ return versions.toArray(new CryptoProtocolVersion[]{});
+ }
+
+ public static HdfsProtos.PerFileEncryptionInfoProto convertPerFileEncInfo(
+ FileEncryptionInfo info) {
+ if (info == null) {
+ return null;
+ }
+ return HdfsProtos.PerFileEncryptionInfoProto.newBuilder()
+ .setKey(getByteString(info.getEncryptedDataEncryptionKey()))
+ .setIv(getByteString(info.getIV()))
+ .setEzKeyVersionName(info.getEzKeyVersionName())
+ .build();
+ }
+
+ public static HdfsProtos.ZoneEncryptionInfoProto convert(
+ CipherSuite suite, CryptoProtocolVersion version, String keyName) {
+ if (suite == null || version == null || keyName == null) {
+ return null;
+ }
+ return HdfsProtos.ZoneEncryptionInfoProto.newBuilder()
+ .setSuite(convert(suite))
+ .setCryptoProtocolVersion(convert(version))
+ .setKeyName(keyName)
+ .build();
+ }
+
+ public static FileEncryptionInfo convert(
+ HdfsProtos.PerFileEncryptionInfoProto fileProto,
+ CipherSuite suite, CryptoProtocolVersion version, String keyName) {
+ if (fileProto == null || suite == null || version == null ||
+ keyName == null) {
+ return null;
+ }
+ byte[] key = fileProto.getKey().toByteArray();
+ byte[] iv = fileProto.getIv().toByteArray();
+ String ezKeyVersionName = fileProto.getEzKeyVersionName();
+ return new FileEncryptionInfo(suite, version, key, iv, keyName,
+ ezKeyVersionName);
+ }
+
+ public static DatanodeInfo[] convert(DatanodeInfosProto datanodeInfosProto) {
+ List<DatanodeInfoProto> proto = datanodeInfosProto.getDatanodesList();
+ DatanodeInfo[] infos = new DatanodeInfo[proto.size()];
+ for (int i = 0; i < infos.length; i++) {
+ infos[i] = convert(proto.get(i));
+ }
+ return infos;
+ }
+
+ static List<DatanodeInfosProto> convert(DatanodeInfo[][] targets) {
+ DatanodeInfosProto[] ret = new DatanodeInfosProto[targets.length];
+ for (int i = 0; i < targets.length; i++) {
+ ret[i] = DatanodeInfosProto.newBuilder()
+ .addAllDatanodes(convert(targets[i])).build();
+ }
+ return Arrays.asList(ret);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 65cde45..944986c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -938,6 +938,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9101. Remove deprecated NameNode.getUri() static helper method.
(Mingliang Liu via wheat9)
+ HDFS-9111. Move hdfs-client protobuf convert methods from PBHelper to
+ PBHelperClient. (Mingliang Liu via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
[20/32] hadoop git commit: YARN-3866. AM-RM protocol changes to
support container resizing. Contributed by Meng Ding
Posted by wa...@apache.org.
YARN-3866. AM-RM protocol changes to support container resizing. Contributed by Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9ffaf3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9ffaf3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9ffaf3a
Branch: refs/heads/YARN-1197
Commit: e9ffaf3ae90663d82eaacb1bc60504a5cf916939
Parents: cc2b473
Author: Jian He <ji...@apache.org>
Authored: Mon Jul 13 17:34:26 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:27 2015 -0700
----------------------------------------------------------------------
.../app/local/TestLocalContainerAllocator.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/protocolrecords/AllocateRequest.java | 57 ++++--
.../api/protocolrecords/AllocateResponse.java | 42 +++--
.../records/ContainerResourceChangeRequest.java | 117 +++++++++++++
.../api/records/ContainerResourceDecrease.java | 78 ---------
.../api/records/ContainerResourceIncrease.java | 84 ---------
.../ContainerResourceIncreaseRequest.java | 80 ---------
.../yarn/api/records/ContainerStatus.java | 13 ++
.../src/main/proto/yarn_protos.proto | 14 +-
.../src/main/proto/yarn_service_protos.proto | 16 +-
.../impl/pb/AllocateRequestPBImpl.java | 119 ++++++++++---
.../impl/pb/AllocateResponsePBImpl.java | 175 +++++--------------
.../ContainerResourceChangeRequestPBImpl.java | 141 +++++++++++++++
.../pb/ContainerResourceDecreasePBImpl.java | 136 --------------
.../pb/ContainerResourceIncreasePBImpl.java | 171 ------------------
.../ContainerResourceIncreaseRequestPBImpl.java | 141 ---------------
.../records/impl/pb/ContainerStatusPBImpl.java | 31 +++-
.../hadoop/yarn/api/TestAllocateRequest.java | 73 --------
.../hadoop/yarn/api/TestAllocateResponse.java | 114 ------------
.../yarn/api/TestContainerResourceDecrease.java | 66 -------
.../yarn/api/TestContainerResourceIncrease.java | 74 --------
.../TestContainerResourceIncreaseRequest.java | 68 -------
.../hadoop/yarn/api/TestPBImplRecords.java | 34 +---
24 files changed, 534 insertions(+), 1319 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index f901ed8..167d804 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -46,8 +46,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -254,8 +252,8 @@ public class TestLocalContainerAllocator {
Resources.none(), null, 1, null,
Collections.<NMToken>emptyList(),
yarnToken,
- Collections.<ContainerResourceIncrease>emptyList(),
- Collections.<ContainerResourceDecrease>emptyList());
+ Collections.<Container>emptyList(),
+ Collections.<Container>emptyList());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 329e743..b9890a7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -200,6 +200,9 @@ Release 2.8.0 - UNRELEASED
YARN-3212. RMNode State Transition Update with DECOMMISSIONING state.
(Junping Du via wangda)
+ YARN-3866. AM-RM protocol changes to support container resizing. (Meng Ding
+ via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index 2458d9b..0b65e5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -22,11 +22,12 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.util.Records;
@@ -46,6 +47,16 @@ import org.apache.hadoop.yarn.util.Records;
* <li>
* A list of unused {@link Container} which are being returned.
* </li>
+ * <li>
+ * A list of {@link ContainerResourceChangeRequest} to inform
+ * the <code>ResourceManager</code> about the resource increase
+ * requirements of running containers.
+ * </li>
+ * <li>
+ * A list of {@link ContainerResourceChangeRequest} to inform
+ * the <code>ResourceManager</code> about the resource decrease
+ * requirements of running containers.
+ * </li>
* </ul>
*
* @see ApplicationMasterProtocol#allocate(AllocateRequest)
@@ -61,7 +72,7 @@ public abstract class AllocateRequest {
List<ContainerId> containersToBeReleased,
ResourceBlacklistRequest resourceBlacklistRequest) {
return newInstance(responseID, appProgress, resourceAsk,
- containersToBeReleased, resourceBlacklistRequest, null);
+ containersToBeReleased, resourceBlacklistRequest, null, null);
}
@Public
@@ -70,7 +81,8 @@ public abstract class AllocateRequest {
List<ResourceRequest> resourceAsk,
List<ContainerId> containersToBeReleased,
ResourceBlacklistRequest resourceBlacklistRequest,
- List<ContainerResourceIncreaseRequest> increaseRequests) {
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
allocateRequest.setResponseId(responseID);
allocateRequest.setProgress(appProgress);
@@ -78,6 +90,7 @@ public abstract class AllocateRequest {
allocateRequest.setReleaseList(containersToBeReleased);
allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
allocateRequest.setIncreaseRequests(increaseRequests);
+ allocateRequest.setDecreaseRequests(decreaseRequests);
return allocateRequest;
}
@@ -184,20 +197,38 @@ public abstract class AllocateRequest {
ResourceBlacklistRequest resourceBlacklistRequest);
/**
- * Get the <code>ContainerResourceIncreaseRequest</code> being sent by the
- * <code>ApplicationMaster</code>
+ * Get the list of container resource increase requests being sent by the
+ * <code>ApplicationMaster</code>.
*/
@Public
- @Stable
- public abstract List<ContainerResourceIncreaseRequest> getIncreaseRequests();
-
+ @Unstable
+ public abstract List<ContainerResourceChangeRequest> getIncreaseRequests();
+
/**
- * Set the <code>ContainerResourceIncreaseRequest</code> to inform the
- * <code>ResourceManager</code> about some container's resources need to be
- * increased
+ * Set the list of container resource increase requests to inform the
+ * <code>ResourceManager</code> about the containers whose resources need
+ * to be increased.
*/
@Public
- @Stable
+ @Unstable
public abstract void setIncreaseRequests(
- List<ContainerResourceIncreaseRequest> increaseRequests);
+ List<ContainerResourceChangeRequest> increaseRequests);
+
+ /**
+ * Get the list of container resource decrease requests being sent by the
+ * <code>ApplicationMaster</code>.
+ */
+ @Public
+ @Unstable
+ public abstract List<ContainerResourceChangeRequest> getDecreaseRequests();
+
+ /**
+ * Set the list of container resource decrease requests to inform the
+ * <code>ResourceManager</code> about the containers whose resources need
+ * to be decreased.
+ */
+ @Public
+ @Unstable
+ public abstract void setDecreaseRequests(
+ List<ContainerResourceChangeRequest> decreaseRequests);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index c4fdb79..c363070 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -28,8 +28,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -59,6 +57,14 @@ import org.apache.hadoop.yarn.util.Records;
* <li>The number of available nodes in a cluster.</li>
* <li>A description of resources requested back by the cluster</li>
* <li>AMRMToken, if AMRMToken has been rolled over</li>
+ * <li>
+ * A list of {@link Container} representing the containers
+ * whose resource has been increased.
+ * </li>
+ * <li>
+ * A list of {@link Container} representing the containers
+ * whose resource has been decreased.
+ * </li>
* </ul>
*
* @see ApplicationMasterProtocol#allocate(AllocateRequest)
@@ -94,8 +100,8 @@ public abstract class AllocateResponse {
List<Container> allocatedContainers, List<NodeReport> updatedNodes,
Resource availResources, AMCommand command, int numClusterNodes,
PreemptionMessage preempt, List<NMToken> nmTokens,
- List<ContainerResourceIncrease> increasedContainers,
- List<ContainerResourceDecrease> decreasedContainers) {
+ List<Container> increasedContainers,
+ List<Container> decreasedContainers) {
AllocateResponse response = newInstance(responseId, completedContainers,
allocatedContainers, updatedNodes, availResources, command,
numClusterNodes, preempt, nmTokens);
@@ -111,8 +117,8 @@ public abstract class AllocateResponse {
List<Container> allocatedContainers, List<NodeReport> updatedNodes,
Resource availResources, AMCommand command, int numClusterNodes,
PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
- List<ContainerResourceIncrease> increasedContainers,
- List<ContainerResourceDecrease> decreasedContainers) {
+ List<Container> increasedContainers,
+ List<Container> decreasedContainers) {
AllocateResponse response =
newInstance(responseId, completedContainers, allocatedContainers,
updatedNodes, availResources, command, numClusterNodes, preempt,
@@ -263,34 +269,38 @@ public abstract class AllocateResponse {
public abstract void setNMTokens(List<NMToken> nmTokens);
/**
- * Get the list of newly increased containers by <code>ResourceManager</code>
+ * Get the list of newly increased containers by
+ * <code>ResourceManager</code>.
*/
@Public
- @Stable
- public abstract List<ContainerResourceIncrease> getIncreasedContainers();
+ @Unstable
+ public abstract List<Container> getIncreasedContainers();
/**
- * Set the list of newly increased containers by <code>ResourceManager</code>
+ * Set the list of newly increased containers by
+ * <code>ResourceManager</code>.
*/
@Private
@Unstable
public abstract void setIncreasedContainers(
- List<ContainerResourceIncrease> increasedContainers);
+ List<Container> increasedContainers);
/**
- * Get the list of newly decreased containers by <code>NodeManager</code>
+ * Get the list of newly decreased containers by
+ * <code>ResourceManager</code>.
*/
@Public
- @Stable
- public abstract List<ContainerResourceDecrease> getDecreasedContainers();
+ @Unstable
+ public abstract List<Container> getDecreasedContainers();
/**
- * Set the list of newly decreased containers by <code>NodeManager</code>
+ * Set the list of newly decreased containers by
+ * <code>ResourceManager</code>.
*/
@Private
@Unstable
public abstract void setDecreasedContainers(
- List<ContainerResourceDecrease> decreasedContainers);
+ List<Container> decreasedContainers);
/**
* The AMRMToken that belong to this attempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceChangeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceChangeRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceChangeRequest.java
new file mode 100644
index 0000000..117015b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceChangeRequest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@code ContainerResourceChangeRequest} represents the request made by an
+ * application to the {@code ResourceManager} to change resource allocation of
+ * a running {@code Container}.
+ * <p>
+ * It includes:
+ * <ul>
+ * <li>{@link ContainerId} for the container.</li>
+ * <li>
+ * {@link Resource} capability of the container after the resource change
+ * is completed.
+ * </li>
+ * </ul>
+ *
+ * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ */
+@Public
+@Unstable
+public abstract class ContainerResourceChangeRequest {
+
+ @Public
+ @Unstable
+ public static ContainerResourceChangeRequest newInstance(
+ ContainerId existingContainerId, Resource targetCapability) {
+ ContainerResourceChangeRequest context = Records
+ .newRecord(ContainerResourceChangeRequest.class);
+ context.setContainerId(existingContainerId);
+ context.setCapability(targetCapability);
+ return context;
+ }
+
+ /**
+ * Get the <code>ContainerId</code> of the container.
+ * @return <code>ContainerId</code> of the container
+ */
+ @Public
+ @Unstable
+ public abstract ContainerId getContainerId();
+
+ /**
+ * Set the <code>ContainerId</code> of the container.
+ * @param containerId <code>ContainerId</code> of the container
+ */
+ @Public
+ @Unstable
+ public abstract void setContainerId(ContainerId containerId);
+
+ /**
+ * Get the <code>Resource</code> capability of the container.
+ * @return <code>Resource</code> capability of the container
+ */
+ @Public
+ @Unstable
+ public abstract Resource getCapability();
+
+ /**
+ * Set the <code>Resource</code> capability of the container.
+ * @param capability <code>Resource</code> capability of the container
+ */
+ @Public
+ @Unstable
+ public abstract void setCapability(Resource capability);
+
+ @Override
+ public int hashCode() {
+ return getCapability().hashCode() + getContainerId().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof ContainerResourceChangeRequest) {
+ ContainerResourceChangeRequest ctx =
+ (ContainerResourceChangeRequest) other;
+
+ if (getContainerId() == null && ctx.getContainerId() != null) {
+ return false;
+ } else if (!getContainerId().equals(ctx.getContainerId())) {
+ return false;
+ }
+
+ if (getCapability() == null && ctx.getCapability() != null) {
+ return false;
+ } else if (!getCapability().equals(ctx.getCapability())) {
+ return false;
+ }
+
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java
deleted file mode 100644
index d766d92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceDecrease.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * Used by Application Master to ask Node Manager reduce size of a specified
- * container
- */
-public abstract class ContainerResourceDecrease {
- @Public
- public static ContainerResourceDecrease newInstance(
- ContainerId existingContainerId, Resource targetCapability) {
- ContainerResourceDecrease context = Records
- .newRecord(ContainerResourceDecrease.class);
- context.setContainerId(existingContainerId);
- context.setCapability(targetCapability);
- return context;
- }
-
- @Public
- public abstract ContainerId getContainerId();
-
- @Public
- public abstract void setContainerId(ContainerId containerId);
-
- @Public
- public abstract Resource getCapability();
-
- @Public
- public abstract void setCapability(Resource capability);
-
- @Override
- public int hashCode() {
- return getCapability().hashCode() + getContainerId().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other instanceof ContainerResourceDecrease) {
- ContainerResourceDecrease ctx = (ContainerResourceDecrease)other;
-
- if (getContainerId() == null && ctx.getContainerId() != null) {
- return false;
- } else if (!getContainerId().equals(ctx.getContainerId())) {
- return false;
- }
-
- if (getCapability() == null && ctx.getCapability() != null) {
- return false;
- } else if (!getCapability().equals(ctx.getCapability())) {
- return false;
- }
-
- return true;
- } else {
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java
deleted file mode 100644
index f4c1560..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncrease.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * Represent a new increased container accepted by Resource Manager
- */
-public abstract class ContainerResourceIncrease {
- @Public
- public static ContainerResourceIncrease newInstance(
- ContainerId existingContainerId, Resource targetCapability, Token token) {
- ContainerResourceIncrease context = Records
- .newRecord(ContainerResourceIncrease.class);
- context.setContainerId(existingContainerId);
- context.setCapability(targetCapability);
- context.setContainerToken(token);
- return context;
- }
-
- @Public
- public abstract ContainerId getContainerId();
-
- @Public
- public abstract void setContainerId(ContainerId containerId);
-
- @Public
- public abstract Resource getCapability();
-
- @Public
- public abstract void setCapability(Resource capability);
-
- @Public
- public abstract Token getContainerToken();
-
- @Public
- public abstract void setContainerToken(Token token);
-
- @Override
- public int hashCode() {
- return getCapability().hashCode() + getContainerId().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other instanceof ContainerResourceIncrease) {
- ContainerResourceIncrease ctx = (ContainerResourceIncrease)other;
-
- if (getContainerId() == null && ctx.getContainerId() != null) {
- return false;
- } else if (!getContainerId().equals(ctx.getContainerId())) {
- return false;
- }
-
- if (getCapability() == null && ctx.getCapability() != null) {
- return false;
- } else if (!getCapability().equals(ctx.getCapability())) {
- return false;
- }
-
- return true;
- } else {
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java
deleted file mode 100644
index 9e3b640..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerResourceIncreaseRequest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * Used by Application Master, send a container resource increase request to
- * Resource Manager
- */
-@Public
-public abstract class ContainerResourceIncreaseRequest {
- @Public
- public static ContainerResourceIncreaseRequest newInstance(
- ContainerId existingContainerId, Resource targetCapability) {
- ContainerResourceIncreaseRequest context = Records
- .newRecord(ContainerResourceIncreaseRequest.class);
- context.setContainerId(existingContainerId);
- context.setCapability(targetCapability);
- return context;
- }
-
- @Public
- public abstract ContainerId getContainerId();
-
- @Public
- public abstract void setContainerId(ContainerId containerId);
-
- @Public
- public abstract Resource getCapability();
-
- @Public
- public abstract void setCapability(Resource capability);
-
- @Override
- public int hashCode() {
- return getCapability().hashCode() + getContainerId().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other instanceof ContainerResourceIncreaseRequest) {
- ContainerResourceIncreaseRequest ctx =
- (ContainerResourceIncreaseRequest) other;
-
- if (getContainerId() == null && ctx.getContainerId() != null) {
- return false;
- } else if (!getContainerId().equals(ctx.getContainerId())) {
- return false;
- }
-
- if (getCapability() == null && ctx.getCapability() != null) {
- return false;
- } else if (!getCapability().equals(ctx.getCapability())) {
- return false;
- }
-
- return true;
- } else {
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
index 5ccf6dc..2c2238f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@code ContainerState} of the container.</li>
* <li><em>Exit status</em> of a completed container.</li>
* <li><em>Diagnostic</em> message for a failed container.</li>
+ * <li>{@link Resource} allocated to the container.</li>
* </ul>
*/
@Public
@@ -114,4 +115,16 @@ public abstract class ContainerStatus {
@Private
@Unstable
public abstract void setDiagnostics(String diagnostics);
+
+ /**
+ * Get the <code>Resource</code> allocated to the container.
+ * @return <code>Resource</code> allocated to the container
+ */
+ @Public
+ @Unstable
+ public abstract Resource getCapability();
+
+ @Private
+ @Unstable
+ public abstract void setCapability(Resource capability);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 0bccfc4..057aeee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -470,6 +470,7 @@ message ContainerStatusProto {
optional ContainerStateProto state = 2;
optional string diagnostics = 3 [default = "N/A"];
optional int32 exit_status = 4 [default = -1000];
+ optional ResourceProto capability = 5;
}
enum ContainerExitStatusProto {
@@ -479,22 +480,11 @@ enum ContainerExitStatusProto {
DISKS_FAILED = -101;
}
-message ContainerResourceIncreaseRequestProto {
+message ContainerResourceChangeRequestProto {
optional ContainerIdProto container_id = 1;
optional ResourceProto capability = 2;
}
-message ContainerResourceIncreaseProto {
- optional ContainerIdProto container_id = 1;
- optional ResourceProto capability = 2;
- optional hadoop.common.TokenProto container_token = 3;
-}
-
-message ContainerResourceDecreaseProto {
- optional ContainerIdProto container_id = 1;
- optional ResourceProto capability = 2;
-}
-
////////////////////////////////////////////////////////////////////////
////// From common//////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index b0b12d1..ff5a127 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -66,7 +66,8 @@ message AllocateRequestProto {
optional ResourceBlacklistRequestProto blacklist_request = 3;
optional int32 response_id = 4;
optional float progress = 5;
- repeated ContainerResourceIncreaseRequestProto increase_request = 6;
+ repeated ContainerResourceChangeRequestProto increase_request = 6;
+ repeated ContainerResourceChangeRequestProto decrease_request = 7;
}
message NMTokenProto {
@@ -84,8 +85,8 @@ message AllocateResponseProto {
optional int32 num_cluster_nodes = 7;
optional PreemptionMessageProto preempt = 8;
repeated NMTokenProto nm_tokens = 9;
- repeated ContainerResourceIncreaseProto increased_containers = 10;
- repeated ContainerResourceDecreaseProto decreased_containers = 11;
+ repeated ContainerProto increased_containers = 10;
+ repeated ContainerProto decreased_containers = 11;
optional hadoop.common.TokenProto am_rm_token = 12;
}
@@ -286,6 +287,15 @@ message GetContainerStatusesResponseProto {
repeated ContainerExceptionMapProto failed_requests = 2;
}
+message IncreaseContainersResourceRequestProto {
+ repeated hadoop.common.TokenProto increase_containers = 1;
+}
+
+message IncreaseContainersResourceResponseProto {
+ repeated ContainerIdProto succeeded_requests = 1;
+ repeated ContainerExceptionMapProto failed_requests = 2;
+}
+
//////////////////////////////////////////////////////
/////// Application_History_Protocol /////////////////
//////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
index dc11165..d6db32c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
@@ -27,15 +27,15 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreaseRequestPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceChangeRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceChangeRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
@@ -52,7 +52,8 @@ public class AllocateRequestPBImpl extends AllocateRequest {
private List<ResourceRequest> ask = null;
private List<ContainerId> release = null;
- private List<ContainerResourceIncreaseRequest> increaseRequests = null;
+ private List<ContainerResourceChangeRequest> increaseRequests = null;
+ private List<ContainerResourceChangeRequest> decreaseRequests = null;
private ResourceBlacklistRequest blacklistRequest = null;
public AllocateRequestPBImpl() {
@@ -101,6 +102,9 @@ public class AllocateRequestPBImpl extends AllocateRequest {
if (this.increaseRequests != null) {
addIncreaseRequestsToProto();
}
+ if (this.decreaseRequests != null) {
+ addDecreaseRequestsToProto();
+ }
if (this.blacklistRequest != null) {
builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest));
}
@@ -162,14 +166,14 @@ public class AllocateRequestPBImpl extends AllocateRequest {
}
@Override
- public List<ContainerResourceIncreaseRequest> getIncreaseRequests() {
+ public List<ContainerResourceChangeRequest> getIncreaseRequests() {
initIncreaseRequests();
return this.increaseRequests;
}
@Override
public void setIncreaseRequests(
- List<ContainerResourceIncreaseRequest> increaseRequests) {
+ List<ContainerResourceChangeRequest> increaseRequests) {
if (increaseRequests == null) {
return;
}
@@ -177,7 +181,24 @@ public class AllocateRequestPBImpl extends AllocateRequest {
this.increaseRequests.clear();
this.increaseRequests.addAll(increaseRequests);
}
-
+
+ @Override
+ public List<ContainerResourceChangeRequest> getDecreaseRequests() {
+ initDecreaseRequests();
+ return this.decreaseRequests;
+ }
+
+ @Override
+ public void setDecreaseRequests(
+ List<ContainerResourceChangeRequest> decreaseRequests) {
+ if (decreaseRequests == null) {
+ return;
+ }
+ initDecreaseRequests();
+ this.decreaseRequests.clear();
+ this.decreaseRequests.addAll(decreaseRequests);
+ }
+
@Override
public ResourceBlacklistRequest getResourceBlacklistRequest() {
AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
@@ -252,28 +273,42 @@ public class AllocateRequestPBImpl extends AllocateRequest {
return;
}
AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerResourceIncreaseRequestProto> list =
+ List<ContainerResourceChangeRequestProto> list =
p.getIncreaseRequestList();
- this.increaseRequests = new ArrayList<ContainerResourceIncreaseRequest>();
+ this.increaseRequests = new ArrayList<ContainerResourceChangeRequest>();
- for (ContainerResourceIncreaseRequestProto c : list) {
+ for (ContainerResourceChangeRequestProto c : list) {
this.increaseRequests.add(convertFromProtoFormat(c));
}
}
-
+
+ private void initDecreaseRequests() {
+ if (this.decreaseRequests != null) {
+ return;
+ }
+ AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List<ContainerResourceChangeRequestProto> list =
+ p.getDecreaseRequestList();
+ this.decreaseRequests = new ArrayList<>();
+
+ for (ContainerResourceChangeRequestProto c : list) {
+ this.decreaseRequests.add(convertFromProtoFormat(c));
+ }
+ }
+
private void addIncreaseRequestsToProto() {
maybeInitBuilder();
builder.clearIncreaseRequest();
if (increaseRequests == null) {
return;
}
- Iterable<ContainerResourceIncreaseRequestProto> iterable =
- new Iterable<ContainerResourceIncreaseRequestProto>() {
+ Iterable<ContainerResourceChangeRequestProto> iterable =
+ new Iterable<ContainerResourceChangeRequestProto>() {
@Override
- public Iterator<ContainerResourceIncreaseRequestProto> iterator() {
- return new Iterator<ContainerResourceIncreaseRequestProto>() {
+ public Iterator<ContainerResourceChangeRequestProto> iterator() {
+ return new Iterator<ContainerResourceChangeRequestProto>() {
- Iterator<ContainerResourceIncreaseRequest> iter =
+ Iterator<ContainerResourceChangeRequest> iter =
increaseRequests.iterator();
@Override
@@ -282,7 +317,7 @@ public class AllocateRequestPBImpl extends AllocateRequest {
}
@Override
- public ContainerResourceIncreaseRequestProto next() {
+ public ContainerResourceChangeRequestProto next() {
return convertToProtoFormat(iter.next());
}
@@ -296,7 +331,43 @@ public class AllocateRequestPBImpl extends AllocateRequest {
};
builder.addAllIncreaseRequest(iterable);
}
-
+
+ private void addDecreaseRequestsToProto() {
+ maybeInitBuilder();
+ builder.clearDecreaseRequest();
+ if (decreaseRequests == null) {
+ return;
+ }
+ Iterable<ContainerResourceChangeRequestProto> iterable =
+ new Iterable<ContainerResourceChangeRequestProto>() {
+ @Override
+ public Iterator<ContainerResourceChangeRequestProto> iterator() {
+ return new Iterator<ContainerResourceChangeRequestProto>() {
+
+ Iterator<ContainerResourceChangeRequest> iter =
+ decreaseRequests.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ContainerResourceChangeRequestProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ }
+ };
+ builder.addAllDecreaseRequest(iterable);
+ }
+
@Override
public List<ContainerId> getReleaseList() {
initReleases();
@@ -367,14 +438,14 @@ public class AllocateRequestPBImpl extends AllocateRequest {
return ((ResourceRequestPBImpl)t).getProto();
}
- private ContainerResourceIncreaseRequestPBImpl convertFromProtoFormat(
- ContainerResourceIncreaseRequestProto p) {
- return new ContainerResourceIncreaseRequestPBImpl(p);
+ private ContainerResourceChangeRequestPBImpl convertFromProtoFormat(
+ ContainerResourceChangeRequestProto p) {
+ return new ContainerResourceChangeRequestPBImpl(p);
}
- private ContainerResourceIncreaseRequestProto convertToProtoFormat(
- ContainerResourceIncreaseRequest t) {
- return ((ContainerResourceIncreaseRequestPBImpl) t).getProto();
+ private ContainerResourceChangeRequestProto convertToProtoFormat(
+ ContainerResourceChangeRequest t) {
+ return ((ContainerResourceChangeRequestPBImpl) t).getProto();
}
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index f2796fd..dd7d1a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -29,8 +29,6 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -38,8 +36,6 @@ import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
@@ -48,8 +44,6 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionMessageProto;
@@ -72,8 +66,8 @@ public class AllocateResponsePBImpl extends AllocateResponse {
private List<Container> allocatedContainers = null;
private List<NMToken> nmTokens = null;
private List<ContainerStatus> completedContainersStatuses = null;
- private List<ContainerResourceIncrease> increasedContainers = null;
- private List<ContainerResourceDecrease> decreasedContainers = null;
+ private List<Container> increasedContainers = null;
+ private List<Container> decreasedContainers = null;
private List<NodeReport> updatedNodes = null;
private PreemptionMessage preempt;
@@ -147,14 +141,14 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
if (this.increasedContainers != null) {
builder.clearIncreasedContainers();
- Iterable<ContainerResourceIncreaseProto> iterable =
- getIncreaseProtoIterable(this.increasedContainers);
+ Iterable<ContainerProto> iterable =
+ getContainerProtoIterable(this.increasedContainers);
builder.addAllIncreasedContainers(iterable);
}
if (this.decreasedContainers != null) {
builder.clearDecreasedContainers();
- Iterable<ContainerResourceDecreaseProto> iterable =
- getChangeProtoIterable(this.decreasedContainers);
+ Iterable<ContainerProto> iterable =
+ getContainerProtoIterable(this.decreasedContainers);
builder.addAllDecreasedContainers(iterable);
}
if (this.amrmToken != null) {
@@ -262,6 +256,36 @@ public class AllocateResponsePBImpl extends AllocateResponse {
allocatedContainers.addAll(containers);
}
+ @Override
+ public synchronized List<Container> getIncreasedContainers() {
+ initLocalIncreasedContainerList();
+ return this.increasedContainers;
+ }
+
+ @Override
+ public synchronized void setIncreasedContainers(
+ final List<Container> containers) {
+ if (containers == null)
+ return;
+ initLocalIncreasedContainerList();
+ increasedContainers.addAll(containers);
+ }
+
+ @Override
+ public synchronized List<Container> getDecreasedContainers() {
+ initLocalDecreasedContainerList();
+ return this.decreasedContainers;
+ }
+
+ @Override
+ public synchronized void setDecreasedContainers(
+ final List<Container> containers) {
+ if (containers == null)
+ return;
+ initLocalDecreasedContainerList();
+ decreasedContainers.addAll(containers);
+ }
+
//// Finished containers
@Override
public synchronized List<ContainerStatus> getCompletedContainersStatuses() {
@@ -333,37 +357,6 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
@Override
- public synchronized List<ContainerResourceIncrease> getIncreasedContainers() {
- initLocalIncreasedContainerList();
- return increasedContainers;
- }
-
- @Override
- public synchronized void setIncreasedContainers(
- List<ContainerResourceIncrease> increasedContainers) {
- if (increasedContainers == null)
- return;
- initLocalIncreasedContainerList();
- this.increasedContainers.addAll(increasedContainers);
- }
-
- @Override
- public synchronized List<ContainerResourceDecrease> getDecreasedContainers() {
- initLocalDecreasedContainerList();
- return decreasedContainers;
- }
-
- @Override
- public synchronized void setDecreasedContainers(
- List<ContainerResourceDecrease> decreasedContainers) {
- if (decreasedContainers == null) {
- return;
- }
- initLocalDecreasedContainerList();
- this.decreasedContainers.addAll(decreasedContainers);
- }
-
- @Override
public synchronized Token getAMRMToken() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
if (amrmToken != null) {
@@ -390,10 +383,10 @@ public class AllocateResponsePBImpl extends AllocateResponse {
return;
}
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerResourceIncreaseProto> list = p.getIncreasedContainersList();
- increasedContainers = new ArrayList<ContainerResourceIncrease>();
+ List<ContainerProto> list = p.getIncreasedContainersList();
+ increasedContainers = new ArrayList<>();
- for (ContainerResourceIncreaseProto c : list) {
+ for (ContainerProto c : list) {
increasedContainers.add(convertFromProtoFormat(c));
}
}
@@ -403,10 +396,10 @@ public class AllocateResponsePBImpl extends AllocateResponse {
return;
}
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerResourceDecreaseProto> list = p.getDecreasedContainersList();
- decreasedContainers = new ArrayList<ContainerResourceDecrease>();
+ List<ContainerProto> list = p.getDecreasedContainersList();
+ decreasedContainers = new ArrayList<>();
- for (ContainerResourceDecreaseProto c : list) {
+ for (ContainerProto c : list) {
decreasedContainers.add(convertFromProtoFormat(c));
}
}
@@ -453,70 +446,6 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
}
- private synchronized Iterable<ContainerResourceIncreaseProto>
- getIncreaseProtoIterable(
- final List<ContainerResourceIncrease> newContainersList) {
- maybeInitBuilder();
- return new Iterable<ContainerResourceIncreaseProto>() {
- @Override
- public synchronized Iterator<ContainerResourceIncreaseProto> iterator() {
- return new Iterator<ContainerResourceIncreaseProto>() {
-
- Iterator<ContainerResourceIncrease> iter = newContainersList
- .iterator();
-
- @Override
- public synchronized boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override
- public synchronized ContainerResourceIncreaseProto next() {
- return convertToProtoFormat(iter.next());
- }
-
- @Override
- public synchronized void remove() {
- throw new UnsupportedOperationException();
- }
- };
-
- }
- };
- }
-
- private synchronized Iterable<ContainerResourceDecreaseProto>
- getChangeProtoIterable(
- final List<ContainerResourceDecrease> newContainersList) {
- maybeInitBuilder();
- return new Iterable<ContainerResourceDecreaseProto>() {
- @Override
- public synchronized Iterator<ContainerResourceDecreaseProto> iterator() {
- return new Iterator<ContainerResourceDecreaseProto>() {
-
- Iterator<ContainerResourceDecrease> iter = newContainersList
- .iterator();
-
- @Override
- public synchronized boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override
- public synchronized ContainerResourceDecreaseProto next() {
- return convertToProtoFormat(iter.next());
- }
-
- @Override
- public synchronized void remove() {
- throw new UnsupportedOperationException();
- }
- };
-
- }
- };
- }
-
private synchronized Iterable<ContainerProto> getContainerProtoIterable(
final List<Container> newContainersList) {
maybeInitBuilder();
@@ -654,26 +583,6 @@ public class AllocateResponsePBImpl extends AllocateResponse {
completedContainersStatuses.add(convertFromProtoFormat(c));
}
}
-
- private synchronized ContainerResourceIncrease convertFromProtoFormat(
- ContainerResourceIncreaseProto p) {
- return new ContainerResourceIncreasePBImpl(p);
- }
-
- private synchronized ContainerResourceIncreaseProto convertToProtoFormat(
- ContainerResourceIncrease t) {
- return ((ContainerResourceIncreasePBImpl) t).getProto();
- }
-
- private synchronized ContainerResourceDecrease convertFromProtoFormat(
- ContainerResourceDecreaseProto p) {
- return new ContainerResourceDecreasePBImpl(p);
- }
-
- private synchronized ContainerResourceDecreaseProto convertToProtoFormat(
- ContainerResourceDecrease t) {
- return ((ContainerResourceDecreasePBImpl) t).getProto();
- }
private synchronized NodeReportPBImpl convertFromProtoFormat(
NodeReportProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceChangeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceChangeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceChangeRequestPBImpl.java
new file mode 100644
index 0000000..f382b8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceChangeRequestPBImpl.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceChangeRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceChangeRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+
+
+public class ContainerResourceChangeRequestPBImpl extends
+ ContainerResourceChangeRequest {
+ ContainerResourceChangeRequestProto proto =
+ ContainerResourceChangeRequestProto.getDefaultInstance();
+ ContainerResourceChangeRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ContainerId existingContainerId = null;
+ private Resource targetCapability = null;
+
+ public ContainerResourceChangeRequestPBImpl() {
+ builder = ContainerResourceChangeRequestProto.newBuilder();
+ }
+
+ public ContainerResourceChangeRequestPBImpl(
+ ContainerResourceChangeRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ContainerResourceChangeRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public ContainerId getContainerId() {
+ ContainerResourceChangeRequestProtoOrBuilder p = viaProto ? proto
+ : builder;
+ if (this.existingContainerId != null) {
+ return this.existingContainerId;
+ }
+ if (p.hasContainerId()) {
+ this.existingContainerId = convertFromProtoFormat(p.getContainerId());
+ }
+ return this.existingContainerId;
+ }
+
+ @Override
+ public void setContainerId(ContainerId existingContainerId) {
+ maybeInitBuilder();
+ if (existingContainerId == null) {
+ builder.clearContainerId();
+ }
+ this.existingContainerId = existingContainerId;
+ }
+
+ @Override
+ public Resource getCapability() {
+ ContainerResourceChangeRequestProtoOrBuilder p = viaProto ? proto
+ : builder;
+ if (this.targetCapability != null) {
+ return this.targetCapability;
+ }
+ if (p.hasCapability()) {
+ this.targetCapability = convertFromProtoFormat(p.getCapability());
+ }
+ return this.targetCapability;
+ }
+
+ @Override
+ public void setCapability(Resource targetCapability) {
+ maybeInitBuilder();
+ if (targetCapability == null) {
+ builder.clearCapability();
+ }
+ this.targetCapability = targetCapability;
+ }
+
+ private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
+ return new ContainerIdPBImpl(p);
+ }
+
+ private ContainerIdProto convertToProtoFormat(ContainerId t) {
+ return ((ContainerIdPBImpl) t).getProto();
+ }
+
+ private Resource convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ((ResourcePBImpl) t).getProto();
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ContainerResourceChangeRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.existingContainerId != null) {
+ builder.setContainerId(convertToProtoFormat(this.existingContainerId));
+ }
+ if (this.targetCapability != null) {
+ builder.setCapability(convertToProtoFormat(this.targetCapability));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java
deleted file mode 100644
index 1834132..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceDecreasePBImpl.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-
-public class ContainerResourceDecreasePBImpl extends ContainerResourceDecrease {
- ContainerResourceDecreaseProto proto = ContainerResourceDecreaseProto
- .getDefaultInstance();
- ContainerResourceDecreaseProto.Builder builder = null;
- boolean viaProto = false;
-
- private ContainerId existingContainerId = null;
- private Resource targetCapability = null;
-
- public ContainerResourceDecreasePBImpl() {
- builder = ContainerResourceDecreaseProto.newBuilder();
- }
-
- public ContainerResourceDecreasePBImpl(ContainerResourceDecreaseProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public ContainerResourceDecreaseProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public ContainerId getContainerId() {
- ContainerResourceDecreaseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.existingContainerId != null) {
- return this.existingContainerId;
- }
- if (p.hasContainerId()) {
- this.existingContainerId = convertFromProtoFormat(p.getContainerId());
- }
- return this.existingContainerId;
- }
-
- @Override
- public void setContainerId(ContainerId existingContainerId) {
- maybeInitBuilder();
- if (existingContainerId == null) {
- builder.clearContainerId();
- }
- this.existingContainerId = existingContainerId;
- }
-
- @Override
- public Resource getCapability() {
- ContainerResourceDecreaseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.targetCapability != null) {
- return this.targetCapability;
- }
- if (p.hasCapability()) {
- this.targetCapability = convertFromProtoFormat(p.getCapability());
- }
- return this.targetCapability;
- }
-
- @Override
- public void setCapability(Resource targetCapability) {
- maybeInitBuilder();
- if (targetCapability == null) {
- builder.clearCapability();
- }
- this.targetCapability = targetCapability;
- }
-
- private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
- return new ContainerIdPBImpl(p);
- }
-
- private ContainerIdProto convertToProtoFormat(ContainerId t) {
- return ((ContainerIdPBImpl) t).getProto();
- }
-
- private Resource convertFromProtoFormat(ResourceProto p) {
- return new ResourcePBImpl(p);
- }
-
- private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl) t).getProto();
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ContainerResourceDecreaseProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private void mergeLocalToBuilder() {
- if (this.existingContainerId != null) {
- builder.setContainerId(convertToProtoFormat(this.existingContainerId));
- }
- if (this.targetCapability != null) {
- builder.setCapability(convertToProtoFormat(this.targetCapability));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java
deleted file mode 100644
index 4e4f3a7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreasePBImpl.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records.impl.pb;
-
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-
-public class ContainerResourceIncreasePBImpl extends ContainerResourceIncrease {
- ContainerResourceIncreaseProto proto = ContainerResourceIncreaseProto
- .getDefaultInstance();
- ContainerResourceIncreaseProto.Builder builder = null;
- boolean viaProto = false;
-
- private ContainerId existingContainerId = null;
- private Resource targetCapability = null;
- private Token token = null;
-
- public ContainerResourceIncreasePBImpl() {
- builder = ContainerResourceIncreaseProto.newBuilder();
- }
-
- public ContainerResourceIncreasePBImpl(ContainerResourceIncreaseProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public ContainerResourceIncreaseProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public ContainerId getContainerId() {
- ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.existingContainerId != null) {
- return this.existingContainerId;
- }
- if (p.hasContainerId()) {
- this.existingContainerId = convertFromProtoFormat(p.getContainerId());
- }
- return this.existingContainerId;
- }
-
- @Override
- public void setContainerId(ContainerId existingContainerId) {
- maybeInitBuilder();
- if (existingContainerId == null) {
- builder.clearContainerId();
- }
- this.existingContainerId = existingContainerId;
- }
-
- @Override
- public Resource getCapability() {
- ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.targetCapability != null) {
- return this.targetCapability;
- }
- if (p.hasCapability()) {
- this.targetCapability = convertFromProtoFormat(p.getCapability());
- }
- return this.targetCapability;
- }
-
- @Override
- public void setCapability(Resource targetCapability) {
- maybeInitBuilder();
- if (targetCapability == null) {
- builder.clearCapability();
- }
- this.targetCapability = targetCapability;
- }
-
- @Override
- public Token getContainerToken() {
- ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.token != null) {
- return this.token;
- }
- if (p.hasContainerToken()) {
- this.token = convertFromProtoFormat(p.getContainerToken());
- }
- return this.token;
- }
-
- @Override
- public void setContainerToken(Token token) {
- maybeInitBuilder();
- if (token == null) {
- builder.clearContainerToken();
- }
- this.token = token;
- }
-
- private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
- return new ContainerIdPBImpl(p);
- }
-
- private ContainerIdProto convertToProtoFormat(ContainerId t) {
- return ((ContainerIdPBImpl) t).getProto();
- }
-
- private Resource convertFromProtoFormat(ResourceProto p) {
- return new ResourcePBImpl(p);
- }
-
- private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl) t).getProto();
- }
-
- private Token convertFromProtoFormat(TokenProto p) {
- return new TokenPBImpl(p);
- }
-
- private TokenProto convertToProtoFormat(Token t) {
- return ((TokenPBImpl) t).getProto();
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ContainerResourceIncreaseProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private void mergeLocalToBuilder() {
- if (this.existingContainerId != null) {
- builder.setContainerId(convertToProtoFormat(this.existingContainerId));
- }
- if (this.targetCapability != null) {
- builder.setCapability(convertToProtoFormat(this.targetCapability));
- }
- if (this.token != null) {
- builder.setContainerToken(convertToProtoFormat(this.token));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java
deleted file mode 100644
index f5ebf6c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerResourceIncreaseRequestPBImpl.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records.impl.pb;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseRequestProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-
-
-public class ContainerResourceIncreaseRequestPBImpl extends
- ContainerResourceIncreaseRequest {
- ContainerResourceIncreaseRequestProto proto =
- ContainerResourceIncreaseRequestProto.getDefaultInstance();
- ContainerResourceIncreaseRequestProto.Builder builder = null;
- boolean viaProto = false;
-
- private ContainerId existingContainerId = null;
- private Resource targetCapability = null;
-
- public ContainerResourceIncreaseRequestPBImpl() {
- builder = ContainerResourceIncreaseRequestProto.newBuilder();
- }
-
- public ContainerResourceIncreaseRequestPBImpl(
- ContainerResourceIncreaseRequestProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public ContainerResourceIncreaseRequestProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public ContainerId getContainerId() {
- ContainerResourceIncreaseRequestProtoOrBuilder p = viaProto ? proto
- : builder;
- if (this.existingContainerId != null) {
- return this.existingContainerId;
- }
- if (p.hasContainerId()) {
- this.existingContainerId = convertFromProtoFormat(p.getContainerId());
- }
- return this.existingContainerId;
- }
-
- @Override
- public void setContainerId(ContainerId existingContainerId) {
- maybeInitBuilder();
- if (existingContainerId == null) {
- builder.clearContainerId();
- }
- this.existingContainerId = existingContainerId;
- }
-
- @Override
- public Resource getCapability() {
- ContainerResourceIncreaseRequestProtoOrBuilder p = viaProto ? proto
- : builder;
- if (this.targetCapability != null) {
- return this.targetCapability;
- }
- if (p.hasCapability()) {
- this.targetCapability = convertFromProtoFormat(p.getCapability());
- }
- return this.targetCapability;
- }
-
- @Override
- public void setCapability(Resource targetCapability) {
- maybeInitBuilder();
- if (targetCapability == null) {
- builder.clearCapability();
- }
- this.targetCapability = targetCapability;
- }
-
- private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
- return new ContainerIdPBImpl(p);
- }
-
- private ContainerIdProto convertToProtoFormat(ContainerId t) {
- return ((ContainerIdPBImpl) t).getProto();
- }
-
- private Resource convertFromProtoFormat(ResourceProto p) {
- return new ResourcePBImpl(p);
- }
-
- private ResourceProto convertToProtoFormat(Resource t) {
- return ((ResourcePBImpl) t).getProto();
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ContainerResourceIncreaseRequestProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private void mergeLocalToBuilder() {
- if (this.existingContainerId != null) {
- builder.setContainerId(convertToProtoFormat(this.existingContainerId));
- }
- if (this.targetCapability != null) {
- builder.setCapability(convertToProtoFormat(this.targetCapability));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 86f2af9..d33d06d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
@@ -78,6 +80,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
sb.append("ContainerStatus: [");
sb.append("ContainerId: ").append(getContainerId()).append(", ");
sb.append("State: ").append(getState()).append(", ");
+ sb.append("Capability: ").append(getCapability()).append(", ");
sb.append("Diagnostics: ").append(getDiagnostics()).append(", ");
sb.append("ExitStatus: ").append(getExitStatus()).append(", ");
sb.append("]");
@@ -168,6 +171,25 @@ public class ContainerStatusPBImpl extends ContainerStatus {
builder.setDiagnostics(diagnostics);
}
+ @Override
+ public synchronized Resource getCapability() {
+ ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasCapability()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getCapability());
+ }
+
+ @Override
+ public synchronized void setCapability(Resource capability) {
+ maybeInitBuilder();
+ if (capability == null) {
+ builder.clearCapability();
+ return;
+ }
+ builder.setCapability(convertToProtoFormat(capability));
+ }
+
private ContainerStateProto convertToProtoFormat(ContainerState e) {
return ProtoUtils.convertToProtoFormat(e);
}
@@ -184,6 +206,11 @@ public class ContainerStatusPBImpl extends ContainerStatus {
return ((ContainerIdPBImpl)t).getProto();
}
+ private ResourceProto convertToProtoFormat(Resource e) {
+ return ((ResourcePBImpl)e).getProto();
+ }
-
-}
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateRequest.java
deleted file mode 100644
index 5ea29f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateRequest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncreaseRequest;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
-import org.junit.Test;
-
-public class TestAllocateRequest {
- @Test
- public void testAllcoateRequestWithIncrease() {
- List<ContainerResourceIncreaseRequest> incRequests =
- new ArrayList<ContainerResourceIncreaseRequest>();
- for (int i = 0; i < 3; i++) {
- incRequests.add(ContainerResourceIncreaseRequest.newInstance(null,
- Resource.newInstance(0, i)));
- }
- AllocateRequest r =
- AllocateRequest.newInstance(123, 0f, null, null, null, incRequests);
-
- // serde
- AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
- r = new AllocateRequestPBImpl(p);
-
- // check value
- Assert.assertEquals(123, r.getResponseId());
- Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size());
-
- for (int i = 0; i < incRequests.size(); i++) {
- Assert.assertEquals(r.getIncreaseRequests().get(i).getCapability()
- .getVirtualCores(), incRequests.get(i).getCapability()
- .getVirtualCores());
- }
- }
-
- @Test
- public void testAllcoateRequestWithoutIncrease() {
- AllocateRequest r =
- AllocateRequest.newInstance(123, 0f, null, null, null, null);
-
- // serde
- AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
- r = new AllocateRequestPBImpl(p);
-
- // check value
- Assert.assertEquals(123, r.getResponseId());
- Assert.assertEquals(0, r.getIncreaseRequests().size());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ffaf3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
deleted file mode 100644
index fbe9af9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl;
-import org.apache.hadoop.yarn.api.records.AMCommand;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NMToken;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-public class TestAllocateResponse {
- @SuppressWarnings("deprecation")
- @Test
- public void testAllocateResponseWithIncDecContainers() {
- List<ContainerResourceIncrease> incContainers =
- new ArrayList<ContainerResourceIncrease>();
- List<ContainerResourceDecrease> decContainers =
- new ArrayList<ContainerResourceDecrease>();
- for (int i = 0; i < 3; i++) {
- incContainers.add(ContainerResourceIncrease.newInstance(null,
- Resource.newInstance(1024, i), null));
- }
- for (int i = 0; i < 5; i++) {
- decContainers.add(ContainerResourceDecrease.newInstance(null,
- Resource.newInstance(1024, i)));
- }
-
- AllocateResponse r =
- AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
- new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
- AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(),
- incContainers, decContainers);
-
- // serde
- AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
- r = new AllocateResponsePBImpl(p);
-
- // check value
- Assert
- .assertEquals(incContainers.size(), r.getIncreasedContainers().size());
- Assert
- .assertEquals(decContainers.size(), r.getDecreasedContainers().size());
-
- for (int i = 0; i < incContainers.size(); i++) {
- Assert.assertEquals(i, r.getIncreasedContainers().get(i).getCapability()
- .getVirtualCores());
- }
-
- for (int i = 0; i < decContainers.size(); i++) {
- Assert.assertEquals(i, r.getDecreasedContainers().get(i).getCapability()
- .getVirtualCores());
- }
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void testAllocateResponseWithoutIncDecContainers() {
- AllocateResponse r =
- AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
- new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
- AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(), null, null);
-
- // serde
- AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
- r = new AllocateResponsePBImpl(p);
-
- // check value
- Assert.assertEquals(0, r.getIncreasedContainers().size());
- Assert.assertEquals(0, r.getDecreasedContainers().size());
- }
-}
[31/32] hadoop git commit: YARN-1651. CapacityScheduler side changes
to support container resize. Contributed by Wangda Tan
Posted by wa...@apache.org.
YARN-1651. CapacityScheduler side changes to support container resize. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/874cc98e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/874cc98e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/874cc98e
Branch: refs/heads/YARN-1197
Commit: 874cc98e60c4756022cff67352f0d9294f853f3a
Parents: d076b7b
Author: Jian He <ji...@apache.org>
Authored: Tue Sep 15 10:21:39 2015 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:29 2015 -0700
----------------------------------------------------------------------
.../v2/app/rm/TestRMContainerAllocator.java | 19 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 14 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 13 +
.../sls/scheduler/ResourceSchedulerWrapper.java | 21 +-
.../sls/scheduler/SLSCapacityScheduler.java | 19 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/impl/TestAMRMClientOnRMRestart.java | 8 +-
.../resource/DefaultResourceCalculator.java | 5 +
.../resource/DominantResourceCalculator.java | 6 +
.../yarn/util/resource/ResourceCalculator.java | 5 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../util/resource/TestResourceCalculator.java | 30 +-
.../protocolrecords/NodeHeartbeatResponse.java | 5 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 5 +-
.../ApplicationMasterService.java | 22 +-
.../server/resourcemanager/RMAuditLogger.java | 2 +
.../server/resourcemanager/RMServerUtils.java | 164 ++++
.../resourcemanager/ResourceTrackerService.java | 7 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 4 +-
.../rmcontainer/RMContainer.java | 4 +
.../RMContainerChangeResourceEvent.java | 44 +
.../rmcontainer/RMContainerEventType.java | 13 +-
.../rmcontainer/RMContainerImpl.java | 121 ++-
.../RMContainerUpdatesAcquiredEvent.java | 35 +
.../server/resourcemanager/rmnode/RMNode.java | 9 +
.../rmnode/RMNodeDecreaseContainerEvent.java | 39 +
.../resourcemanager/rmnode/RMNodeEventType.java | 1 +
.../resourcemanager/rmnode/RMNodeImpl.java | 93 ++
.../rmnode/RMNodeStatusEvent.java | 32 +-
.../scheduler/AbstractYarnScheduler.java | 150 ++-
.../resourcemanager/scheduler/Allocation.java | 22 +-
.../scheduler/AppSchedulingInfo.java | 249 ++++-
.../resourcemanager/scheduler/QueueMetrics.java | 16 +-
.../scheduler/SchedContainerChangeRequest.java | 118 +++
.../scheduler/SchedulerApplication.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 253 +++--
.../scheduler/SchedulerNode.java | 31 +
.../scheduler/SchedulerUtils.java | 11 +-
.../scheduler/YarnScheduler.java | 14 +-
.../scheduler/capacity/AbstractCSQueue.java | 23 +-
.../scheduler/capacity/CSAssignment.java | 9 +
.../scheduler/capacity/CSQueue.java | 16 +
.../scheduler/capacity/CapacityScheduler.java | 83 +-
.../scheduler/capacity/LeafQueue.java | 127 ++-
.../scheduler/capacity/ParentQueue.java | 115 ++-
.../allocator/AbstractContainerAllocator.java | 131 +++
.../capacity/allocator/ContainerAllocator.java | 149 +--
.../allocator/IncreaseContainerAllocator.java | 365 +++++++
.../allocator/RegularContainerAllocator.java | 30 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 68 +-
.../scheduler/fair/FairScheduler.java | 35 +-
.../scheduler/fifo/FifoScheduler.java | 25 +-
.../server/resourcemanager/Application.java | 2 +-
.../yarn/server/resourcemanager/MockAM.java | 9 +
.../yarn/server/resourcemanager/MockNodes.java | 13 +
.../yarn/server/resourcemanager/MockRM.java | 13 +
.../TestApplicationMasterService.java | 144 ++-
.../applicationsmanager/TestAMRestart.java | 15 +-
.../TestRMAppLogAggregationStatus.java | 10 +-
.../attempt/TestRMAppAttemptTransitions.java | 32 +-
.../rmcontainer/TestRMContainerImpl.java | 117 ++-
.../capacity/TestCapacityScheduler.java | 128 ++-
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerAllocation.java | 50 +-
.../capacity/TestContainerResizing.java | 963 +++++++++++++++++++
.../scheduler/capacity/TestLeafQueue.java | 4 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
.../scheduler/capacity/TestReservations.java | 9 +-
.../scheduler/fair/FairSchedulerTestBase.java | 6 +-
.../fair/TestContinuousScheduling.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 30 +-
.../scheduler/fifo/TestFifoScheduler.java | 28 +-
72 files changed, 3861 insertions(+), 512 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 1a3829e..e6aebb4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -98,6 +98,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
@@ -1634,8 +1635,10 @@ public class TestRMContainerAllocator {
@Override
public synchronized Allocation allocate(
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
- List<ContainerId> release,
- List<String> blacklistAdditions, List<String> blacklistRemovals) {
+ List<ContainerId> release, List<String> blacklistAdditions,
+ List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy = ResourceRequest.newInstance(req
@@ -1649,8 +1652,8 @@ public class TestRMContainerAllocator {
lastBlacklistAdditions = blacklistAdditions;
lastBlacklistRemovals = blacklistRemovals;
return super.allocate(
- applicationAttemptId, askCopy, release,
- blacklistAdditions, blacklistRemovals);
+ applicationAttemptId, askCopy, release, blacklistAdditions,
+ blacklistRemovals, increaseRequests, decreaseRequests);
}
}
@@ -1670,8 +1673,10 @@ public class TestRMContainerAllocator {
@Override
public synchronized Allocation allocate(
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
- List<ContainerId> release,
- List<String> blacklistAdditions, List<String> blacklistRemovals) {
+ List<ContainerId> release, List<String> blacklistAdditions,
+ List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequest,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy = ResourceRequest.newInstance(req
@@ -1682,7 +1687,7 @@ public class TestRMContainerAllocator {
SecurityUtil.setTokenServiceUseIp(false);
Allocation normalAlloc = super.allocate(
applicationAttemptId, askCopy, release,
- blacklistAdditions, blacklistRemovals);
+ blacklistAdditions, blacklistRemovals, null, null);
List<Container> containers = normalAlloc.getContainers();
if(containers.size() > 0) {
// allocate excess container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 2d2c3e0..dae2ce7 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -174,6 +175,19 @@ public class NodeInfo {
public Set<String> getNodeLabels() {
return RMNodeLabelsManager.EMPTY_STRING_SET;
}
+
+ @Override
+ public void updateNodeHeartbeatResponseForContainersDecreasing(
+ NodeHeartbeatResponse response) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public List<Container> pullNewlyIncreasedContainers() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
public static RMNode newNodeInfo(String rackName, String hostName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index ecc4734..8c65ccc 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -163,4 +164,16 @@ public class RMNodeWrapper implements RMNode {
public Set<String> getNodeLabels() {
return RMNodeLabelsManager.EMPTY_STRING_SET;
}
+
+ @Override
+ public void updateNodeHeartbeatResponseForContainersDecreasing(
+ NodeHeartbeatResponse response) {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public List<Container> pullNewlyIncreasedContainers() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index 14e2645..310b3b5 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -72,6 +73,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -202,15 +204,16 @@ final public class ResourceSchedulerWrapper
@Override
public Allocation allocate(ApplicationAttemptId attemptId,
- List<ResourceRequest> resourceRequests,
- List<ContainerId> containerIds,
- List<String> strings, List<String> strings2) {
+ List<ResourceRequest> resourceRequests, List<ContainerId> containerIds,
+ List<String> strings, List<String> strings2,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
if (metricsON) {
final Timer.Context context = schedulerAllocateTimer.time();
Allocation allocation = null;
try {
allocation = scheduler.allocate(attemptId, resourceRequests,
- containerIds, strings, strings2);
+ containerIds, strings, strings2, null, null);
return allocation;
} finally {
context.stop();
@@ -224,7 +227,7 @@ final public class ResourceSchedulerWrapper
}
} else {
return scheduler.allocate(attemptId,
- resourceRequests, containerIds, strings, strings2);
+ resourceRequests, containerIds, strings, strings2, null, null);
}
}
@@ -959,4 +962,12 @@ final public class ResourceSchedulerWrapper
return Priority.newInstance(0);
}
+ @Override
+ protected void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest,
+ SchedulerApplicationAttempt attempt) {
+ // TODO Auto-generated method stub
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index a4416db..3626027 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -176,15 +177,17 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
@Override
public Allocation allocate(ApplicationAttemptId attemptId,
- List<ResourceRequest> resourceRequests,
- List<ContainerId> containerIds,
- List<String> strings, List<String> strings2) {
+ List<ResourceRequest> resourceRequests, List<ContainerId> containerIds,
+ List<String> strings, List<String> strings2,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
if (metricsON) {
final Timer.Context context = schedulerAllocateTimer.time();
Allocation allocation = null;
try {
- allocation = super.allocate(attemptId, resourceRequests,
- containerIds, strings, strings2);
+ allocation = super
+ .allocate(attemptId, resourceRequests, containerIds, strings,
+ strings2, increaseRequests, decreaseRequests);
return allocation;
} finally {
context.stop();
@@ -197,8 +200,8 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
}
} else {
- return super.allocate(attemptId,
- resourceRequests, containerIds, strings, strings2);
+ return super.allocate(attemptId, resourceRequests, containerIds, strings,
+ strings2, increaseRequests, decreaseRequests);
}
}
@@ -426,7 +429,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
if (pool != null) pool.shutdown();
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({ "unchecked", "rawtypes" })
private void initMetrics() throws Exception {
metrics = new MetricRegistry();
// configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2f055ca..196f727 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -220,6 +220,9 @@ Release 2.8.0 - UNRELEASED
YARN-3868. Recovery support for container resizing. (Meng Ding via jianhe)
+ YARN-1651. CapacityScheduler side changes to support container resize.
+ (Wangda Tan via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index 108ad37..2394747 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -525,7 +526,9 @@ public class TestAMRMClientOnRMRestart {
public synchronized Allocation allocate(
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
List<ContainerId> release, List<String> blacklistAdditions,
- List<String> blacklistRemovals) {
+ List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
for (ResourceRequest req : ask) {
ResourceRequest reqCopy =
@@ -539,7 +542,8 @@ public class TestAMRMClientOnRMRestart {
lastBlacklistAdditions = blacklistAdditions;
lastBlacklistRemovals = blacklistRemovals;
return super.allocate(applicationAttemptId, askCopy, release,
- blacklistAdditions, blacklistRemovals);
+ blacklistAdditions, blacklistRemovals, increaseRequests,
+ decreaseRequests);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index c2fc1f0..2fdf214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -110,4 +110,9 @@ public class DefaultResourceCalculator extends ResourceCalculator {
);
}
+ @Override
+ public boolean fitsIn(Resource cluster,
+ Resource smaller, Resource bigger) {
+ return smaller.getMemory() <= bigger.getMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 2ee95ce..b5c9967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -209,4 +209,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
);
}
+ @Override
+ public boolean fitsIn(Resource cluster,
+ Resource smaller, Resource bigger) {
+ return smaller.getMemory() <= bigger.getMemory()
+ && smaller.getVirtualCores() <= bigger.getVirtualCores();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 442196c..3a31225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -171,4 +171,9 @@ public abstract class ResourceCalculator {
*/
public abstract Resource divideAndCeil(Resource numerator, int denominator);
+ /**
+ * Check if a smaller resource can be contained by bigger resource.
+ */
+ public abstract boolean fitsIn(Resource cluster,
+ Resource smaller, Resource bigger);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 503d456..b05d021 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -267,6 +267,11 @@ public class Resources {
return smaller.getMemory() <= bigger.getMemory() &&
smaller.getVirtualCores() <= bigger.getVirtualCores();
}
+
+ public static boolean fitsIn(ResourceCalculator rc, Resource cluster,
+ Resource smaller, Resource bigger) {
+ return rc.fitsIn(cluster, smaller, bigger);
+ }
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
return createResource(Math.min(lhs.getMemory(), rhs.getMemory()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
index 6a0b62e..0654891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -41,6 +41,35 @@ public class TestResourceCalculator {
public TestResourceCalculator(ResourceCalculator rs) {
this.resourceCalculator = rs;
}
+
+ @Test(timeout = 10000)
+ public void testFitsIn() {
+ Resource cluster = Resource.newInstance(1024, 1);
+
+ if (resourceCalculator instanceof DefaultResourceCalculator) {
+ Assert.assertTrue(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(2, 1)));
+ Assert.assertTrue(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(2, 2)));
+ Assert.assertTrue(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(1, 2)));
+ Assert.assertTrue(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(1, 1)));
+ Assert.assertFalse(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(2, 1), Resource.newInstance(1, 2)));
+ } else if (resourceCalculator instanceof DominantResourceCalculator) {
+ Assert.assertFalse(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(2, 1)));
+ Assert.assertTrue(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(2, 2)));
+ Assert.assertTrue(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(1, 2)));
+ Assert.assertFalse(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(1, 2), Resource.newInstance(1, 1)));
+ Assert.assertFalse(resourceCalculator.fitsIn(cluster,
+ Resource.newInstance(2, 1), Resource.newInstance(1, 2)));
+ }
+ }
@Test(timeout = 10000)
public void testResourceCalculatorCompareMethod() {
@@ -92,7 +121,6 @@ public class TestResourceCalculator {
}
-
private void assertResourcesOperations(Resource clusterResource,
Resource lhs, Resource rhs, boolean lessThan, boolean lessThanOrEqual,
boolean greaterThan, boolean greaterThanOrEqual, Resource max,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 38fbc82..c0ccf57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -19,12 +19,13 @@
package org.apache.hadoop.yarn.server.api.protocolrecords;
import java.nio.ByteBuffer;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -73,5 +74,5 @@ public interface NodeHeartbeatResponse {
void setAreNodeLabelsAcceptedByRM(boolean areNodeLabelsAcceptedByRM);
List<Container> getContainersToDecrease();
- void addAllContainersToDecrease(List<Container> containersToDecrease);
+ void addAllContainersToDecrease(Collection<Container> containersToDecrease);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 12c5230..dc65141 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -20,14 +20,15 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
@@ -437,7 +438,7 @@ public class NodeHeartbeatResponsePBImpl extends
@Override
public void addAllContainersToDecrease(
- final List<Container> containersToDecrease) {
+ final Collection<Container> containersToDecrease) {
if (containersToDecrease == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 14142de..87c7bfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -451,11 +451,13 @@ public class ApplicationMasterService extends AbstractService implements
req.setNodeLabelExpression(asc.getNodeLabelExpression());
}
}
+
+ Resource maximumCapacity = rScheduler.getMaximumResourceCapability();
// sanity check
try {
RMServerUtils.normalizeAndValidateRequests(ask,
- rScheduler.getMaximumResourceCapability(), app.getQueue(),
+ maximumCapacity, app.getQueue(),
rScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
@@ -469,6 +471,15 @@ public class ApplicationMasterService extends AbstractService implements
throw e;
}
+ try {
+ RMServerUtils.increaseDecreaseRequestSanityCheck(rmContext,
+ request.getIncreaseRequests(), request.getDecreaseRequests(),
+ maximumCapacity);
+ } catch (InvalidResourceRequestException e) {
+ LOG.warn(e);
+ throw e;
+ }
+
// In the case of work-preserving AM restart, it's possible for the
// AM to release containers from the earlier attempt.
if (!app.getApplicationSubmissionContext()
@@ -493,8 +504,9 @@ public class ApplicationMasterService extends AbstractService implements
allocation = EMPTY_ALLOCATION;
} else {
allocation =
- this.rScheduler.allocate(appAttemptId, ask, release,
- blacklistAdditions, blacklistRemovals);
+ this.rScheduler.allocate(appAttemptId, ask, release,
+ blacklistAdditions, blacklistRemovals,
+ request.getIncreaseRequests(), request.getDecreaseRequests());
}
if (!blacklistAdditions.isEmpty() || !blacklistRemovals.isEmpty()) {
@@ -540,6 +552,10 @@ public class ApplicationMasterService extends AbstractService implements
.pullJustFinishedContainers());
allocateResponse.setResponseId(lastResponse.getResponseId() + 1);
allocateResponse.setAvailableResources(allocation.getResourceLimit());
+
+ // Handling increased/decreased containers
+ allocateResponse.setIncreasedContainers(allocation.getIncreasedContainers());
+ allocateResponse.setDecreasedContainers(allocation.getDecreasedContainers());
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index f049d97..cd9a61d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -56,6 +56,8 @@ public class RMAuditLogger {
public static final String RELEASE_CONTAINER = "AM Released Container";
public static final String UPDATE_APP_PRIORITY =
"Update Application Priority Request";
+ public static final String CHANGE_CONTAINER_RESOURCE =
+ "AM Changed Container Resource";
// Some commonly used descriptions
public static final String UNAUTHORIZED_USER = "Unauthorized user";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 4d2e41c..cc30593 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -22,8 +22,10 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
@@ -34,6 +36,7 @@ import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -49,10 +52,14 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -107,6 +114,89 @@ public class RMServerUtils {
queueName, scheduler, rmContext, queueInfo);
}
}
+
+ /**
+ * Normalize container increase/decrease request, it will normalize and update
+ * ContainerResourceChangeRequest.targetResource
+ *
+ * <pre>
+ * - Throw exception when any other error happens
+ * </pre>
+ */
+ public static void checkAndNormalizeContainerChangeRequest(
+ RMContext rmContext, ContainerResourceChangeRequest request,
+ boolean increase) throws InvalidResourceRequestException {
+ ContainerId containerId = request.getContainerId();
+ ResourceScheduler scheduler = rmContext.getScheduler();
+ RMContainer rmContainer = scheduler.getRMContainer(containerId);
+ ResourceCalculator rc = scheduler.getResourceCalculator();
+
+ if (null == rmContainer) {
+ String msg =
+ "Failed to get rmContainer for "
+ + (increase ? "increase" : "decrease")
+ + " request, with container-id=" + containerId;
+ throw new InvalidResourceRequestException(msg);
+ }
+
+ if (rmContainer.getState() != RMContainerState.RUNNING) {
+ String msg =
+ "rmContainer's state is not RUNNING, for "
+ + (increase ? "increase" : "decrease")
+ + " request, with container-id=" + containerId;
+ throw new InvalidResourceRequestException(msg);
+ }
+
+ Resource targetResource = Resources.normalize(rc, request.getCapability(),
+ scheduler.getMinimumResourceCapability(),
+ scheduler.getMaximumResourceCapability(),
+ scheduler.getMinimumResourceCapability());
+
+ // Compare targetResource and original resource
+ Resource originalResource = rmContainer.getAllocatedResource();
+
+ // Resource comparasion should be >= (or <=) for all resource vectors, for
+ // example, you cannot request target resource of a <10G, 10> container to
+ // <20G, 8>
+ if (increase) {
+ if (originalResource.getMemory() > targetResource.getMemory()
+ || originalResource.getVirtualCores() > targetResource
+ .getVirtualCores()) {
+ String msg =
+ "Trying to increase a container, but target resource has some"
+ + " resource < original resource, target=" + targetResource
+ + " original=" + originalResource + " containerId="
+ + containerId;
+ throw new InvalidResourceRequestException(msg);
+ }
+ } else {
+ if (originalResource.getMemory() < targetResource.getMemory()
+ || originalResource.getVirtualCores() < targetResource
+ .getVirtualCores()) {
+ String msg =
+ "Trying to decrease a container, but target resource has "
+ + "some resource > original resource, target=" + targetResource
+ + " original=" + originalResource + " containerId="
+ + containerId;
+ throw new InvalidResourceRequestException(msg);
+ }
+ }
+
+ RMNode rmNode = rmContext.getRMNodes().get(rmContainer.getAllocatedNode());
+
+ // Target resource of the increase request is more than NM can offer
+ if (!Resources.fitsIn(scheduler.getResourceCalculator(),
+ scheduler.getClusterResource(), targetResource,
+ rmNode.getTotalCapability())) {
+ String msg = "Target resource=" + targetResource + " of containerId="
+ + containerId + " is more than node's total resource="
+ + rmNode.getTotalCapability();
+ throw new InvalidResourceRequestException(msg);
+ }
+
+ // Update normalized target resource
+ request.setCapability(targetResource);
+ }
/*
* @throw <code>InvalidResourceBlacklistRequestException </code> if the
@@ -123,6 +213,80 @@ public class RMServerUtils {
}
}
}
+
+ /**
+ * Check if we have:
+ * - Request for same containerId and different target resource
+ * - If targetResources violates maximum/minimumAllocation
+ */
+ public static void increaseDecreaseRequestSanityCheck(RMContext rmContext,
+ List<ContainerResourceChangeRequest> incRequests,
+ List<ContainerResourceChangeRequest> decRequests,
+ Resource maximumAllocation) throws InvalidResourceRequestException {
+ checkDuplicatedIncreaseDecreaseRequest(incRequests, decRequests);
+ validateIncreaseDecreaseRequest(rmContext, incRequests, maximumAllocation,
+ true);
+ validateIncreaseDecreaseRequest(rmContext, decRequests, maximumAllocation,
+ false);
+ }
+
+ private static void checkDuplicatedIncreaseDecreaseRequest(
+ List<ContainerResourceChangeRequest> incRequests,
+ List<ContainerResourceChangeRequest> decRequests)
+ throws InvalidResourceRequestException {
+ String msg = "There're multiple increase or decrease container requests "
+ + "for same containerId=";
+ Set<ContainerId> existedContainerIds = new HashSet<ContainerId>();
+ if (incRequests != null) {
+ for (ContainerResourceChangeRequest r : incRequests) {
+ if (!existedContainerIds.add(r.getContainerId())) {
+ throw new InvalidResourceRequestException(msg + r.getContainerId());
+ }
+ }
+ }
+
+ if (decRequests != null) {
+ for (ContainerResourceChangeRequest r : decRequests) {
+ if (!existedContainerIds.add(r.getContainerId())) {
+ throw new InvalidResourceRequestException(msg + r.getContainerId());
+ }
+ }
+ }
+ }
+
+ private static void validateIncreaseDecreaseRequest(RMContext rmContext,
+ List<ContainerResourceChangeRequest> requests, Resource maximumAllocation,
+ boolean increase)
+ throws InvalidResourceRequestException {
+ if (requests == null) {
+ return;
+ }
+ for (ContainerResourceChangeRequest request : requests) {
+ if (request.getCapability().getMemory() < 0
+ || request.getCapability().getMemory() > maximumAllocation
+ .getMemory()) {
+ throw new InvalidResourceRequestException("Invalid "
+ + (increase ? "increase" : "decrease") + " request"
+ + ", requested memory < 0"
+ + ", or requested memory > max configured" + ", requestedMemory="
+ + request.getCapability().getMemory() + ", maxMemory="
+ + maximumAllocation.getMemory());
+ }
+ if (request.getCapability().getVirtualCores() < 0
+ || request.getCapability().getVirtualCores() > maximumAllocation
+ .getVirtualCores()) {
+ throw new InvalidResourceRequestException("Invalid "
+ + (increase ? "increase" : "decrease") + " request"
+ + ", requested virtual cores < 0"
+ + ", or requested virtual cores > max configured"
+ + ", requestedVirtualCores="
+ + request.getCapability().getVirtualCores() + ", maxVirtualCores="
+ + maximumAllocation.getVirtualCores());
+ }
+
+ checkAndNormalizeContainerChangeRequest(rmContext, request, increase);
+ }
+ }
/**
* It will validate to make sure all the containers belong to correct
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 7e774c5..248cdc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -452,6 +452,8 @@ public class ResourceTrackerService extends AbstractService implements
getResponseId() + 1, NodeAction.NORMAL, null, null, null, null,
nextHeartBeatInterval);
rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse);
+ rmNode.updateNodeHeartbeatResponseForContainersDecreasing(
+ nodeHeartBeatResponse);
populateKeys(request, nodeHeartBeatResponse);
@@ -464,8 +466,9 @@ public class ResourceTrackerService extends AbstractService implements
// 4. Send status to RMNode, saving the latest response.
RMNodeStatusEvent nodeStatusEvent =
new RMNodeStatusEvent(nodeId, remoteNodeStatus.getNodeHealthStatus(),
- remoteNodeStatus.getContainersStatuses(),
- remoteNodeStatus.getKeepAliveApplications(), nodeHeartBeatResponse);
+ remoteNodeStatus.getContainersStatuses(),
+ remoteNodeStatus.getKeepAliveApplications(), nodeHeartBeatResponse,
+ remoteNodeStatus.getIncreasedContainers());
if (request.getLogAggregationReportsForApps() != null
&& !request.getLogAggregationReportsForApps().isEmpty()) {
nodeStatusEvent.setLogAggregationReportsForApps(request
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 629b2a3..43de3ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -971,7 +971,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
Collections.singletonList(appAttempt.amReq),
EMPTY_CONTAINER_RELEASE_LIST,
amBlacklist.getAdditions(),
- amBlacklist.getRemovals());
+ amBlacklist.getRemovals(), null, null);
if (amContainerAllocation != null
&& amContainerAllocation.getContainers() != null) {
assert (amContainerAllocation.getContainers().size() == 0);
@@ -995,7 +995,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
Allocation amContainerAllocation =
appAttempt.scheduler.allocate(appAttempt.applicationAttemptId,
EMPTY_CONTAINER_REQUEST_LIST, EMPTY_CONTAINER_RELEASE_LIST, null,
- null);
+ null, null, null);
// There must be at least one container allocated, because a
// CONTAINER_ALLOCATED is emitted after an RMContainer is constructed,
// and is put in SchedulerApplication#newlyAllocatedContainers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index 21d79ee..dc0d9ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -82,4 +82,8 @@ public interface RMContainer extends EventHandler<RMContainerEvent> {
String getNodeHttpAddress();
String getNodeLabelExpression();
+
+ boolean hasIncreaseReservation();
+
+ void cancelIncreaseReservation();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java
new file mode 100644
index 0000000..920cfdb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerChangeResourceEvent.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public class RMContainerChangeResourceEvent extends RMContainerEvent {
+
+ final Resource targetResource;
+ final boolean increase;
+
+ public RMContainerChangeResourceEvent(ContainerId containerId,
+ Resource targetResource, boolean increase) {
+ super(containerId, RMContainerEventType.CHANGE_RESOURCE);
+
+ this.targetResource = targetResource;
+ this.increase = increase;
+ }
+
+ public Resource getTargetResource() {
+ return targetResource;
+ }
+
+ public boolean isIncrease() {
+ return increase;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerEventType.java
index 259d68b3..a3b4b76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerEventType.java
@@ -25,6 +25,10 @@ public enum RMContainerEventType {
ACQUIRED,
KILL, // Also from Node on NodeRemoval
RESERVED,
+
+ // when a container acquired by AM after
+ // it increased/decreased
+ ACQUIRE_UPDATED_CONTAINER,
LAUNCHED,
FINISHED,
@@ -35,5 +39,12 @@ public enum RMContainerEventType {
// Source: ContainerAllocationExpirer
EXPIRE,
- RECOVER
+ RECOVER,
+
+ // Source: Scheduler
+ // Resource change approved by scheduler
+ CHANGE_RESOURCE,
+
+ // NM reported resource change is done
+ NM_DONE_CHANGE_RESOURCE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index a3d8bee..8133657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -118,7 +118,18 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
.addTransition(RMContainerState.RUNNING, RMContainerState.RELEASED,
RMContainerEventType.RELEASED, new KillTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
- RMContainerEventType.EXPIRE)
+ RMContainerEventType.RESERVED, new ContainerReservedTransition())
+ .addTransition(RMContainerState.RUNNING, RMContainerState.EXPIRED,
+ RMContainerEventType.EXPIRE,
+ new ContainerExpiredWhileRunningTransition())
+ .addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
+ RMContainerEventType.CHANGE_RESOURCE, new ChangeResourceTransition())
+ .addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
+ RMContainerEventType.ACQUIRE_UPDATED_CONTAINER,
+ new ContainerAcquiredWhileRunningTransition())
+ .addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
+ RMContainerEventType.NM_DONE_CHANGE_RESOURCE,
+ new NMReportedContainerChangeIsDoneTransition())
// Transitions from COMPLETED state
.addTransition(RMContainerState.COMPLETED, RMContainerState.COMPLETED,
@@ -140,9 +151,7 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
RMContainerEventType.KILL, RMContainerEventType.FINISHED))
// create the topology tables
- .installTopology();
-
-
+ .installTopology();
private final StateMachine<RMContainerState, RMContainerEventType,
RMContainerEvent> stateMachine;
@@ -166,6 +175,8 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
private ContainerStatus finishedStatus;
private boolean isAMContainer;
private List<ResourceRequest> resourceRequests;
+
+ private volatile boolean hasIncreaseReservation = false;
public RMContainerImpl(Container container,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
@@ -264,7 +275,12 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
@Override
public Resource getAllocatedResource() {
- return container.getResource();
+ try {
+ readLock.lock();
+ return container.getResource();
+ } finally {
+ readLock.unlock();
+ }
}
@Override
@@ -471,8 +487,8 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
}
}
- private static final class ContainerReservedTransition extends
- BaseTransition {
+ private static final class ContainerReservedTransition
+ extends BaseTransition {
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
@@ -480,6 +496,12 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
container.reservedResource = e.getReservedResource();
container.reservedNode = e.getReservedNode();
container.reservedPriority = e.getReservedPriority();
+
+ if (!EnumSet.of(RMContainerState.NEW, RMContainerState.RESERVED)
+ .contains(container.getState())) {
+ // When container's state != NEW/RESERVED, it is an increase reservation
+ container.hasIncreaseReservation = true;
+ }
}
}
@@ -509,6 +531,70 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
.getApplicationAttemptId().getApplicationId(), container.nodeId));
}
}
+
+ private static final class ContainerAcquiredWhileRunningTransition extends
+ BaseTransition {
+
+ @Override
+ public void transition(RMContainerImpl container, RMContainerEvent event) {
+ RMContainerUpdatesAcquiredEvent acquiredEvent =
+ (RMContainerUpdatesAcquiredEvent) event;
+ if (acquiredEvent.isIncreasedContainer()) {
+ // If container is increased but not acquired by AM, we will start
+ // containerAllocationExpirer for this container in this transition.
+ container.containerAllocationExpirer.register(event.getContainerId());
+ }
+ }
+ }
+
+ private static final class NMReportedContainerChangeIsDoneTransition
+ extends BaseTransition {
+
+ @Override
+ public void transition(RMContainerImpl container, RMContainerEvent event) {
+ // Unregister the allocation expirer, it is already increased..
+ container.containerAllocationExpirer.unregister(event.getContainerId());
+ }
+ }
+
+ private static final class ContainerExpiredWhileRunningTransition extends
+ BaseTransition {
+
+ @Override
+ public void transition(RMContainerImpl container, RMContainerEvent event) {
+ // When the container expired, and it has a pending increased request, we
+ // will kill the container.
+ // TODO, we can do better for this: roll back container resource to the
+ // resource before increase, and notify scheduler about this decrease as
+ // well. Will do that in a separated JIRA.
+ new KillTransition().transition(container, event);
+ }
+ }
+
+ private static final class ChangeResourceTransition extends BaseTransition {
+
+ @Override
+ public void transition(RMContainerImpl container, RMContainerEvent event) {
+ RMContainerChangeResourceEvent changeEvent = (RMContainerChangeResourceEvent)event;
+
+ // Register with containerAllocationExpirer.
+ // For now, we assume timeout for increase is as same as container
+ // allocation.
+ if (!changeEvent.isIncrease()) {
+ // if this is a decrease request, if container was increased but not
+ // told to NM, we can consider previous increase is cancelled,
+ // unregister from the containerAllocationExpirer
+ container.containerAllocationExpirer.unregister(container
+ .getContainerId());
+ }
+
+ container.container.setResource(changeEvent.getTargetResource());
+
+ // We reach here means we either allocated increase reservation OR
+ // decreased container, reservation will be cancelled anyway.
+ container.hasIncreaseReservation = false;
+ }
+ }
private static final class ContainerRescheduledTransition extends
FinishedTransition {
@@ -561,13 +647,14 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
RMAppAttempt rmAttempt = container.rmContext.getRMApps()
.get(container.getApplicationAttemptId().getApplicationId())
.getCurrentAppAttempt();
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
- container);
- }
if (rmAttempt != null) {
+ if (ContainerExitStatus.PREEMPTED == container.finishedStatus
+ .getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
+ container);
+ }
+
long usedMillis = container.finishTime - container.creationTime;
long memorySeconds = resource.getMemory()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
@@ -665,4 +752,14 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
}
return -1;
}
+
+ @Override
+ public boolean hasIncreaseReservation() {
+ return hasIncreaseReservation;
+ }
+
+ @Override
+ public void cancelIncreaseReservation() {
+ hasIncreaseReservation = false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerUpdatesAcquiredEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerUpdatesAcquiredEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerUpdatesAcquiredEvent.java
new file mode 100644
index 0000000..0dccc5f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerUpdatesAcquiredEvent.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public class RMContainerUpdatesAcquiredEvent extends RMContainerEvent {
+ private final boolean increasedContainer;
+
+ public RMContainerUpdatesAcquiredEvent(ContainerId containerId,
+ boolean increasedContainer) {
+ super(containerId, RMContainerEventType.ACQUIRE_UPDATED_CONTAINER);
+ this.increasedContainer = increasedContainer;
+ }
+
+ public boolean isIncreasedContainer() {
+ return increasedContainer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 6bb0971..f28422a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -146,4 +147,12 @@ public interface RMNode {
* @return labels in this node
*/
public Set<String> getNodeLabels();
+
+ /**
+ * Update containers to be decreased
+ */
+ public void updateNodeHeartbeatResponseForContainersDecreasing(
+ NodeHeartbeatResponse response);
+
+ public List<Container> pullNewlyIncreasedContainers();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java
new file mode 100644
index 0000000..62925ad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java
@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+public class RMNodeDecreaseContainerEvent extends RMNodeEvent {
+ final List<Container> toBeDecreasedContainers;
+
+ public RMNodeDecreaseContainerEvent(NodeId nodeId,
+ List<Container> toBeDecreasedContainers) {
+ super(nodeId, RMNodeEventType.DECREASE_CONTAINER);
+
+ this.toBeDecreasedContainers = toBeDecreasedContainers;
+ }
+
+ public List<Container> getToBeDecreasedContainers() {
+ return toBeDecreasedContainers;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
index ad36036..abe8544 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
@@ -42,6 +42,7 @@ public enum RMNodeEventType {
// Source: Container
CONTAINER_ALLOCATED,
CLEANUP_CONTAINER,
+ DECREASE_CONTAINER,
// Source: RMAppAttempt
FINISHED_CONTAINERS_PULLED_BY_AM,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 391b6ff..33e4714 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -19,9 +19,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -36,6 +40,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -131,6 +136,12 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/* the list of applications that are running on this node */
private final List<ApplicationId> runningApplications =
new ArrayList<ApplicationId>();
+
+ private final Map<ContainerId, Container> toBeDecreasedContainers =
+ new HashMap<>();
+
+ private final Map<ContainerId, Container> nmReportedIncreasedContainers =
+ new HashMap<>();
private NodeHeartbeatResponse latestNodeHeartBeatResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
@@ -180,6 +191,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
RMNodeEventType.RECONNECTED, new ReconnectNodeTransition())
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.RESOURCE_UPDATE, new UpdateNodeResourceWhenRunningTransition())
+ .addTransition(NodeState.RUNNING, NodeState.RUNNING,
+ RMNodeEventType.DECREASE_CONTAINER,
+ new DecreaseContainersTransition())
.addTransition(NodeState.RUNNING, NodeState.SHUTDOWN,
RMNodeEventType.SHUTDOWN,
new DeactivateNodeTransition(NodeState.SHUTDOWN))
@@ -484,6 +498,24 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.writeLock.unlock();
}
};
+
+ @VisibleForTesting
+ public Collection<Container> getToBeDecreasedContainers() {
+ return toBeDecreasedContainers.values();
+ }
+
+ @Override
+ public void updateNodeHeartbeatResponseForContainersDecreasing(
+ NodeHeartbeatResponse response) {
+ this.writeLock.lock();
+
+ try {
+ response.addAllContainersToDecrease(toBeDecreasedContainers.values());
+ toBeDecreasedContainers.clear();
+ } finally {
+ this.writeLock.unlock();
+ }
+ }
@Override
public NodeHeartbeatResponse getLastNodeHeartBeatResponse() {
@@ -836,6 +868,19 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
RMNodeFinishedContainersPulledByAMEvent) event).getContainers());
}
}
+
+ public static class DecreaseContainersTransition
+ implements SingleArcTransition<RMNodeImpl, RMNodeEvent> {
+
+ @Override
+ public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
+ RMNodeDecreaseContainerEvent de = (RMNodeDecreaseContainerEvent) event;
+
+ for (Container c : de.getToBeDecreasedContainers()) {
+ rmNode.toBeDecreasedContainers.put(c.getId(), c);
+ }
+ }
+ }
public static class DeactivateNodeTransition
implements SingleArcTransition<RMNodeImpl, RMNodeEvent> {
@@ -986,6 +1031,8 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
rmNode.handleContainerStatus(statusEvent.getContainers());
+ rmNode.handleReportedIncreasedContainers(
+ statusEvent.getNMReportedIncreasedContainers());
List<LogAggregationReport> logAggregationReportsForApps =
statusEvent.getLogAggregationReportsForApps();
@@ -1079,6 +1126,34 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
return nlm.getLabelsOnNode(nodeId);
}
+
+ private void handleReportedIncreasedContainers(
+ List<Container> reportedIncreasedContainers) {
+ for (Container container : reportedIncreasedContainers) {
+ ContainerId containerId = container.getId();
+
+ // Don't bother with containers already scheduled for cleanup, or for
+ // applications already killed. The scheduler doens't need to know any
+ // more about this container
+ if (containersToClean.contains(containerId)) {
+ LOG.info("Container " + containerId + " already scheduled for "
+ + "cleanup, no further processing");
+ continue;
+ }
+
+ ApplicationId containerAppId =
+ containerId.getApplicationAttemptId().getApplicationId();
+
+ if (finishedApplications.contains(containerAppId)) {
+ LOG.info("Container " + containerId
+ + " belongs to an application that is already killed,"
+ + " no further processing");
+ continue;
+ }
+
+ this.nmReportedIncreasedContainers.put(containerId, container);
+ }
+ }
private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
// Filter the map to only obtain just launched containers and finished
@@ -1149,4 +1224,22 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
}
+ @Override
+ public List<Container> pullNewlyIncreasedContainers() {
+ try {
+ writeLock.lock();
+
+ if (nmReportedIncreasedContainers.isEmpty()) {
+ return Collections.EMPTY_LIST;
+ } else {
+ List<Container> container =
+ new ArrayList<Container>(nmReportedIncreasedContainers.values());
+ nmReportedIncreasedContainers.clear();
+ return container;
+ }
+
+ } finally {
+ writeLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
index b95d7d3..8323f3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
+import java.util.Collections;
import java.util.List;
+
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
@@ -33,28 +36,36 @@ public class RMNodeStatusEvent extends RMNodeEvent {
private final NodeHeartbeatResponse latestResponse;
private final List<ApplicationId> keepAliveAppIds;
private List<LogAggregationReport> logAggregationReportsForApps;
-
+ private final List<Container> nmReportedIncreasedContainers;
+
+ // Used by tests
public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus,
List<ContainerStatus> collection, List<ApplicationId> keepAliveAppIds,
NodeHeartbeatResponse latestResponse) {
- super(nodeId, RMNodeEventType.STATUS_UPDATE);
- this.nodeHealthStatus = nodeHealthStatus;
- this.containersCollection = collection;
- this.keepAliveAppIds = keepAliveAppIds;
- this.latestResponse = latestResponse;
- this.logAggregationReportsForApps = null;
+ this(nodeId, nodeHealthStatus, collection, keepAliveAppIds,
+ latestResponse, null);
}
public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus,
List<ContainerStatus> collection, List<ApplicationId> keepAliveAppIds,
NodeHeartbeatResponse latestResponse,
- List<LogAggregationReport> logAggregationReportsForApps) {
+ List<Container> nmReportedIncreasedContainers) {
+ this(nodeId, nodeHealthStatus, collection, keepAliveAppIds, latestResponse,
+ null, nmReportedIncreasedContainers);
+ }
+
+ public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus,
+ List<ContainerStatus> collection, List<ApplicationId> keepAliveAppIds,
+ NodeHeartbeatResponse latestResponse,
+ List<LogAggregationReport> logAggregationReportsForApps,
+ List<Container> nmReportedIncreasedContainers) {
super(nodeId, RMNodeEventType.STATUS_UPDATE);
this.nodeHealthStatus = nodeHealthStatus;
this.containersCollection = collection;
this.keepAliveAppIds = keepAliveAppIds;
this.latestResponse = latestResponse;
this.logAggregationReportsForApps = logAggregationReportsForApps;
+ this.nmReportedIncreasedContainers = nmReportedIncreasedContainers;
}
public NodeHealthStatus getNodeHealthStatus() {
@@ -81,4 +92,9 @@ public class RMNodeStatusEvent extends RMNodeEvent {
List<LogAggregationReport> logAggregationReportsForApps) {
this.logAggregationReportsForApps = logAggregationReportsForApps;
}
+
+ public List<Container> getNMReportedIncreasedContainers() {
+ return nmReportedIncreasedContainers == null ? Collections.EMPTY_LIST
+ : nmReportedIncreasedContainers;
+ }
}
\ No newline at end of file
[13/32] hadoop git commit: HDFS-9111. Move hdfs-client protobuf
convert methods from PBHelper to PBHelperClient. Contributed by Mingliang
Liu.
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
deleted file mode 100644
index 23e8f57..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-import java.net.URI;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileEncryptionInfo;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-
-/**
- * Interface that represents the over the wire information
- * including block locations for a file.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class HdfsLocatedFileStatus extends HdfsFileStatus {
- private final LocatedBlocks locations;
-
- /**
- * Constructor
- *
- * @param length size
- * @param isdir if this is directory
- * @param block_replication the file's replication factor
- * @param blocksize the file's block size
- * @param modification_time most recent modification time
- * @param access_time most recent access time
- * @param permission permission
- * @param owner owner
- * @param group group
- * @param symlink symbolic link
- * @param path local path name in java UTF8 format
- * @param fileId the file id
- * @param locations block locations
- * @param feInfo file encryption info
- */
- public HdfsLocatedFileStatus(long length, boolean isdir,
- int block_replication, long blocksize, long modification_time,
- long access_time, FsPermission permission, String owner, String group,
- byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
- int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) {
- super(length, isdir, block_replication, blocksize, modification_time,
- access_time, permission, owner, group, symlink, path, fileId,
- childrenNum, feInfo, storagePolicy);
- this.locations = locations;
- }
-
- public LocatedBlocks getBlockLocations() {
- return locations;
- }
-
- public final LocatedFileStatus makeQualifiedLocated(URI defaultUri,
- Path path) {
- return new LocatedFileStatus(getLen(), isDir(), getReplication(),
- getBlockSize(), getModificationTime(),
- getAccessTime(),
- getPermission(), getOwner(), getGroup(),
- isSymlink() ? new Path(getSymlink()) : null,
- (getFullPath(path)).makeQualified(
- defaultUri, null), // fully-qualify path
- DFSUtilClient.locatedBlocks2Locations(getBlockLocations()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index 85da414..59cf884 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpTransferBlockP
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ReleaseShortCircuitAccessRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmRequestProto;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
@@ -115,7 +114,7 @@ public abstract class Receiver implements DataTransferProtocol {
proto.getClass().getSimpleName());
try {
readBlock(PBHelperClient.convert(proto.getHeader().getBaseHeader().getBlock()),
- PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
+ PBHelperClient.convert(proto.getHeader().getBaseHeader().getToken()),
proto.getHeader().getClientName(),
proto.getOffset(),
proto.getLen(),
@@ -131,17 +130,17 @@ public abstract class Receiver implements DataTransferProtocol {
/** Receive OP_WRITE_BLOCK */
private void opWriteBlock(DataInputStream in) throws IOException {
final OpWriteBlockProto proto = OpWriteBlockProto.parseFrom(vintPrefixed(in));
- final DatanodeInfo[] targets = PBHelper.convert(proto.getTargetsList());
+ final DatanodeInfo[] targets = PBHelperClient.convert(proto.getTargetsList());
TraceScope traceScope = continueTraceSpan(proto.getHeader(),
proto.getClass().getSimpleName());
try {
writeBlock(PBHelperClient.convert(proto.getHeader().getBaseHeader().getBlock()),
PBHelperClient.convertStorageType(proto.getStorageType()),
- PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
+ PBHelperClient.convert(proto.getHeader().getBaseHeader().getToken()),
proto.getHeader().getClientName(),
targets,
- PBHelper.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length),
- PBHelper.convert(proto.getSource()),
+ PBHelperClient.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length),
+ PBHelperClient.convert(proto.getSource()),
fromProto(proto.getStage()),
proto.getPipelineSize(),
proto.getMinBytesRcvd(), proto.getMaxBytesRcvd(),
@@ -152,7 +151,7 @@ public abstract class Receiver implements DataTransferProtocol {
CachingStrategy.newDefaultStrategy()),
(proto.hasAllowLazyPersist() ? proto.getAllowLazyPersist() : false),
(proto.hasPinning() ? proto.getPinning(): false),
- (PBHelper.convertBooleanList(proto.getTargetPinningsList())));
+ (PBHelperClient.convertBooleanList(proto.getTargetPinningsList())));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -162,15 +161,15 @@ public abstract class Receiver implements DataTransferProtocol {
private void opTransferBlock(DataInputStream in) throws IOException {
final OpTransferBlockProto proto =
OpTransferBlockProto.parseFrom(vintPrefixed(in));
- final DatanodeInfo[] targets = PBHelper.convert(proto.getTargetsList());
+ final DatanodeInfo[] targets = PBHelperClient.convert(proto.getTargetsList());
TraceScope traceScope = continueTraceSpan(proto.getHeader(),
proto.getClass().getSimpleName());
try {
transferBlock(PBHelperClient.convert(proto.getHeader().getBaseHeader().getBlock()),
- PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
+ PBHelperClient.convert(proto.getHeader().getBaseHeader().getToken()),
proto.getHeader().getClientName(),
targets,
- PBHelper.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length));
+ PBHelperClient.convertStorageTypes(proto.getTargetStorageTypesList(), targets.length));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -181,12 +180,12 @@ public abstract class Receiver implements DataTransferProtocol {
final OpRequestShortCircuitAccessProto proto =
OpRequestShortCircuitAccessProto.parseFrom(vintPrefixed(in));
SlotId slotId = (proto.hasSlotId()) ?
- PBHelper.convert(proto.getSlotId()) : null;
+ PBHelperClient.convert(proto.getSlotId()) : null;
TraceScope traceScope = continueTraceSpan(proto.getHeader(),
proto.getClass().getSimpleName());
try {
requestShortCircuitFds(PBHelperClient.convert(proto.getHeader().getBlock()),
- PBHelper.convert(proto.getHeader().getToken()),
+ PBHelperClient.convert(proto.getHeader().getToken()),
slotId, proto.getMaxVersion(),
proto.getSupportsReceiptVerification());
} finally {
@@ -202,7 +201,7 @@ public abstract class Receiver implements DataTransferProtocol {
TraceScope traceScope = continueTraceSpan(proto.getTraceInfo(),
proto.getClass().getSimpleName());
try {
- releaseShortCircuitFds(PBHelper.convert(proto.getSlotId()));
+ releaseShortCircuitFds(PBHelperClient.convert(proto.getSlotId()));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -229,9 +228,9 @@ public abstract class Receiver implements DataTransferProtocol {
try {
replaceBlock(PBHelperClient.convert(proto.getHeader().getBlock()),
PBHelperClient.convertStorageType(proto.getStorageType()),
- PBHelper.convert(proto.getHeader().getToken()),
+ PBHelperClient.convert(proto.getHeader().getToken()),
proto.getDelHint(),
- PBHelper.convert(proto.getSource()));
+ PBHelperClient.convert(proto.getSource()));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -244,7 +243,7 @@ public abstract class Receiver implements DataTransferProtocol {
proto.getClass().getSimpleName());
try {
copyBlock(PBHelperClient.convert(proto.getHeader().getBlock()),
- PBHelper.convert(proto.getHeader().getToken()));
+ PBHelperClient.convert(proto.getHeader().getToken()));
} finally {
if (traceScope != null) traceScope.close();
}
@@ -257,7 +256,7 @@ public abstract class Receiver implements DataTransferProtocol {
proto.getClass().getSimpleName());
try {
blockChecksum(PBHelperClient.convert(proto.getHeader().getBlock()),
- PBHelper.convert(proto.getHeader().getToken()));
+ PBHelperClient.convert(proto.getHeader().getToken()));
} finally {
if (traceScope != null) traceScope.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
index 3adb4a8..76ad820 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
@@ -123,7 +123,9 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements
throws ServiceException {
BlockLocalPathInfo resp;
try {
- resp = impl.getBlockLocalPathInfo(PBHelperClient.convert(request.getBlock()), PBHelper.convert(request.getToken()));
+ resp = impl.getBlockLocalPathInfo(
+ PBHelperClient.convert(request.getBlock()),
+ PBHelperClient.convert(request.getToken()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -150,7 +152,7 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements
GetDatanodeInfoResponseProto res;
try {
res = GetDatanodeInfoResponseProto.newBuilder()
- .setLocalInfo(PBHelper.convert(impl.getDatanodeInfo())).build();
+ .setLocalInfo(PBHelperClient.convert(impl.getDatanodeInfo())).build();
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolPB.java
index 0d3796c..e7ce44b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolPB.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ClientNamenodeProtocol;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
@@ -31,7 +31,7 @@ import org.apache.hadoop.security.token.TokenInfo;
@InterfaceAudience.Private
@InterfaceStability.Stable
@KerberosInfo(
- serverPrincipal = DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = HdfsClientConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY)
@TokenInfo(DelegationTokenSelector.class)
@ProtocolInfo(protocolName = HdfsConstants.CLIENT_NAMENODE_PROTOCOL_NAME,
protocolVersion = 1)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 9f26ca3..d93277c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -367,7 +367,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
Builder builder = GetBlockLocationsResponseProto
.newBuilder();
if (b != null) {
- builder.setLocations(PBHelper.convert(b)).build();
+ builder.setLocations(PBHelperClient.convert(b)).build();
}
return builder.build();
} catch (IOException e) {
@@ -382,7 +382,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
try {
FsServerDefaults result = server.getServerDefaults();
return GetServerDefaultsResponseProto.newBuilder()
- .setServerDefaults(PBHelper.convert(result))
+ .setServerDefaults(PBHelperClient.convert(result))
.build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -395,14 +395,14 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
CreateRequestProto req) throws ServiceException {
try {
HdfsFileStatus result = server.create(req.getSrc(),
- PBHelper.convert(req.getMasked()), req.getClientName(),
- PBHelper.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
+ PBHelperClient.convert(req.getMasked()), req.getClientName(),
+ PBHelperClient.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
(short) req.getReplication(), req.getBlockSize(),
- PBHelper.convertCryptoProtocolVersions(
+ PBHelperClient.convertCryptoProtocolVersions(
req.getCryptoProtocolVersionList()));
if (result != null) {
- return CreateResponseProto.newBuilder().setFs(PBHelper.convert(result))
+ return CreateResponseProto.newBuilder().setFs(PBHelperClient.convert(result))
.build();
}
return VOID_CREATE_RESPONSE;
@@ -416,16 +416,16 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
AppendRequestProto req) throws ServiceException {
try {
EnumSetWritable<CreateFlag> flags = req.hasFlag() ?
- PBHelper.convertCreateFlag(req.getFlag()) :
+ PBHelperClient.convertCreateFlag(req.getFlag()) :
new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND));
LastBlockWithStatus result = server.append(req.getSrc(),
req.getClientName(), flags);
AppendResponseProto.Builder builder = AppendResponseProto.newBuilder();
if (result.getLastBlock() != null) {
- builder.setBlock(PBHelper.convert(result.getLastBlock()));
+ builder.setBlock(PBHelperClient.convert(result.getLastBlock()));
}
if (result.getFileStatus() != null) {
- builder.setStat(PBHelper.convert(result.getFileStatus()));
+ builder.setStat(PBHelperClient.convert(result.getFileStatus()));
}
return builder.build();
} catch (IOException e) {
@@ -450,7 +450,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public SetPermissionResponseProto setPermission(RpcController controller,
SetPermissionRequestProto req) throws ServiceException {
try {
- server.setPermission(req.getSrc(), PBHelper.convert(req.getPermission()));
+ server.setPermission(req.getSrc(), PBHelperClient.convert(req.getPermission()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -493,12 +493,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
req.getSrc(),
req.getClientName(),
req.hasPrevious() ? PBHelperClient.convert(req.getPrevious()) : null,
- (excl == null || excl.size() == 0) ? null : PBHelper.convert(excl
+ (excl == null || excl.size() == 0) ? null : PBHelperClient.convert(excl
.toArray(new DatanodeInfoProto[excl.size()])), req.getFileId(),
(favor == null || favor.size() == 0) ? null : favor
.toArray(new String[favor.size()]));
return AddBlockResponseProto.newBuilder()
- .setBlock(PBHelper.convert(result)).build();
+ .setBlock(PBHelperClient.convert(result)).build();
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -514,15 +514,15 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
List<DatanodeInfoProto> excludesList = req.getExcludesList();
LocatedBlock result = server.getAdditionalDatanode(req.getSrc(),
req.getFileId(), PBHelperClient.convert(req.getBlk()),
- PBHelper.convert(existingList.toArray(
+ PBHelperClient.convert(existingList.toArray(
new DatanodeInfoProto[existingList.size()])),
existingStorageIDsList.toArray(
new String[existingStorageIDsList.size()]),
- PBHelper.convert(excludesList.toArray(
- new DatanodeInfoProto[excludesList.size()])),
+ PBHelperClient.convert(excludesList.toArray(
+ new DatanodeInfoProto[excludesList.size()])),
req.getNumAdditionalNodes(), req.getClientName());
return GetAdditionalDatanodeResponseProto.newBuilder().setBlock(
- PBHelper.convert(result))
+ PBHelperClient.convert(result))
.build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -548,8 +548,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
ReportBadBlocksRequestProto req) throws ServiceException {
try {
List<LocatedBlockProto> bl = req.getBlocksList();
- server.reportBadBlocks(PBHelper.convertLocatedBlock(
- bl.toArray(new LocatedBlockProto[bl.size()])));
+ server.reportBadBlocks(PBHelperClient.convertLocatedBlock(
+ bl.toArray(new LocatedBlockProto[bl.size()])));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -620,7 +620,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
MkdirsRequestProto req) throws ServiceException {
try {
boolean result = server.mkdirs(req.getSrc(),
- PBHelper.convert(req.getMasked()), req.getCreateParent());
+ PBHelperClient.convert(req.getMasked()), req.getCreateParent());
return MkdirsResponseProto.newBuilder().setResult(result).build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -636,7 +636,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
req.getNeedLocation());
if (result !=null) {
return GetListingResponseProto.newBuilder().setDirList(
- PBHelper.convert(result)).build();
+ PBHelperClient.convert(result)).build();
} else {
return VOID_GETLISTING_RESPONSE;
}
@@ -684,7 +684,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public GetFsStatsResponseProto getFsStats(RpcController controller,
GetFsStatusRequestProto req) throws ServiceException {
try {
- return PBHelper.convert(server.getStats());
+ return PBHelperClient.convert(server.getStats());
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -696,7 +696,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throws ServiceException {
try {
List<? extends DatanodeInfoProto> result = PBHelperClient.convert(server
- .getDatanodeReport(PBHelper.convert(req.getType())));
+ .getDatanodeReport(PBHelperClient.convert(req.getType())));
return GetDatanodeReportResponseProto.newBuilder()
.addAllDi(result).build();
} catch (IOException e) {
@@ -709,8 +709,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, GetDatanodeStorageReportRequestProto req)
throws ServiceException {
try {
- List<DatanodeStorageReportProto> reports = PBHelper.convertDatanodeStorageReports(
- server.getDatanodeStorageReport(PBHelper.convert(req.getType())));
+ List<DatanodeStorageReportProto> reports = PBHelperClient.convertDatanodeStorageReports(
+ server.getDatanodeStorageReport(PBHelperClient.convert(req.getType())));
return GetDatanodeStorageReportResponseProto.newBuilder()
.addAllDatanodeStorageReports(reports)
.build();
@@ -736,7 +736,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public SetSafeModeResponseProto setSafeMode(RpcController controller,
SetSafeModeRequestProto req) throws ServiceException {
try {
- boolean result = server.setSafeMode(PBHelper.convert(req.getAction()),
+ boolean result = server.setSafeMode(PBHelperClient.convert(req.getAction()),
req.getChecked());
return SetSafeModeResponseProto.newBuilder().setResult(result).build();
} catch (IOException e) {
@@ -799,10 +799,10 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RollingUpgradeRequestProto req) throws ServiceException {
try {
final RollingUpgradeInfo info = server.rollingUpgrade(
- PBHelper.convert(req.getAction()));
+ PBHelperClient.convert(req.getAction()));
final RollingUpgradeResponseProto.Builder b = RollingUpgradeResponseProto.newBuilder();
if (info != null) {
- b.setRollingUpgradeInfo(PBHelper.convert(info));
+ b.setRollingUpgradeInfo(PBHelperClient.convert(info));
}
return b.build();
} catch (IOException e) {
@@ -818,7 +818,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
CorruptFileBlocks result = server.listCorruptFileBlocks(
req.getPath(), req.hasCookie() ? req.getCookie(): null);
return ListCorruptFileBlocksResponseProto.newBuilder()
- .setCorrupt(PBHelper.convert(result))
+ .setCorrupt(PBHelperClient.convert(result))
.build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -845,7 +845,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
if (result != null) {
return GetFileInfoResponseProto.newBuilder().setFs(
- PBHelper.convert(result)).build();
+ PBHelperClient.convert(result)).build();
}
return VOID_GETFILEINFO_RESPONSE;
} catch (IOException e) {
@@ -860,7 +860,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
HdfsFileStatus result = server.getFileLinkInfo(req.getSrc());
if (result != null) {
return GetFileLinkInfoResponseProto.newBuilder().setFs(
- PBHelper.convert(result)).build();
+ PBHelperClient.convert(result)).build();
} else {
return VOID_GETFILELINKINFO_RESPONSE;
}
@@ -877,7 +877,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
try {
ContentSummary result = server.getContentSummary(req.getPath());
return GetContentSummaryResponseProto.newBuilder()
- .setSummary(PBHelper.convert(result)).build();
+ .setSummary(PBHelperClient.convert(result)).build();
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -925,7 +925,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
CreateSymlinkRequestProto req) throws ServiceException {
try {
server.createSymlink(req.getTarget(), req.getLink(),
- PBHelper.convert(req.getDirPerm()), req.getCreateParent());
+ PBHelperClient.convert(req.getDirPerm()), req.getCreateParent());
return VOID_CREATESYMLINK_RESPONSE;
} catch (IOException e) {
throw new ServiceException(e);
@@ -953,7 +953,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, UpdateBlockForPipelineRequestProto req)
throws ServiceException {
try {
- LocatedBlockProto result = PBHelper.convert(server
+ LocatedBlockProto result = PBHelperClient.convert(server
.updateBlockForPipeline(PBHelperClient.convert(req.getBlock()),
req.getClientName()));
return UpdateBlockForPipelineResponseProto.newBuilder().setBlock(result)
@@ -972,7 +972,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
server.updatePipeline(req.getClientName(),
PBHelperClient.convert(req.getOldBlock()),
PBHelperClient.convert(req.getNewBlock()),
- PBHelper.convert(newNodes.toArray(new DatanodeIDProto[newNodes.size()])),
+ PBHelperClient.convert(newNodes.toArray(new DatanodeIDProto[newNodes.size()])),
newStorageIDs.toArray(new String[newStorageIDs.size()]));
return VOID_UPDATEPIPELINE_RESPONSE;
} catch (IOException e) {
@@ -1003,7 +1003,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, RenewDelegationTokenRequestProto req)
throws ServiceException {
try {
- long result = server.renewDelegationToken(PBHelper
+ long result = server.renewDelegationToken(PBHelperClient
.convertDelegationToken(req.getToken()));
return RenewDelegationTokenResponseProto.newBuilder()
.setNewExpiryTime(result).build();
@@ -1017,7 +1017,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, CancelDelegationTokenRequestProto req)
throws ServiceException {
try {
- server.cancelDelegationToken(PBHelper.convertDelegationToken(req
+ server.cancelDelegationToken(PBHelperClient.convertDelegationToken(req
.getToken()));
return VOID_CANCELDELEGATIONTOKEN_RESPONSE;
} catch (IOException e) {
@@ -1046,7 +1046,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
GetDataEncryptionKeyResponseProto.newBuilder();
DataEncryptionKey encryptionKey = server.getDataEncryptionKey();
if (encryptionKey != null) {
- builder.setDataEncryptionKey(PBHelper.convert(encryptionKey));
+ builder.setDataEncryptionKey(PBHelperClient.convert(encryptionKey));
}
return builder.build();
} catch (IOException e) {
@@ -1125,7 +1125,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
.getSnapshottableDirListing();
if (result != null) {
return GetSnapshottableDirListingResponseProto.newBuilder().
- setSnapshottableDirList(PBHelper.convert(result)).build();
+ setSnapshottableDirList(PBHelperClient.convert(result)).build();
} else {
return NULL_GET_SNAPSHOTTABLE_DIR_LISTING_RESPONSE;
}
@@ -1143,7 +1143,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
request.getSnapshotRoot(), request.getFromSnapshot(),
request.getToSnapshot());
return GetSnapshotDiffReportResponseProto.newBuilder()
- .setDiffReport(PBHelper.convert(report)).build();
+ .setDiffReport(PBHelperClient.convert(report)).build();
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1167,8 +1167,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throws ServiceException {
try {
long id = server.addCacheDirective(
- PBHelper.convert(request.getInfo()),
- PBHelper.convertCacheFlags(request.getCacheFlags()));
+ PBHelperClient.convert(request.getInfo()),
+ PBHelperClient.convertCacheFlags(request.getCacheFlags()));
return AddCacheDirectiveResponseProto.newBuilder().
setId(id).build();
} catch (IOException e) {
@@ -1182,8 +1182,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throws ServiceException {
try {
server.modifyCacheDirective(
- PBHelper.convert(request.getInfo()),
- PBHelper.convertCacheFlags(request.getCacheFlags()));
+ PBHelperClient.convert(request.getInfo()),
+ PBHelperClient.convertCacheFlags(request.getCacheFlags()));
return ModifyCacheDirectiveResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -1210,14 +1210,14 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throws ServiceException {
try {
CacheDirectiveInfo filter =
- PBHelper.convert(request.getFilter());
+ PBHelperClient.convert(request.getFilter());
BatchedEntries<CacheDirectiveEntry> entries =
server.listCacheDirectives(request.getPrevId(), filter);
ListCacheDirectivesResponseProto.Builder builder =
ListCacheDirectivesResponseProto.newBuilder();
builder.setHasMore(entries.hasMore());
for (int i=0, n=entries.size(); i<n; i++) {
- builder.addElements(PBHelper.convert(entries.get(i)));
+ builder.addElements(PBHelperClient.convert(entries.get(i)));
}
return builder.build();
} catch (IOException e) {
@@ -1229,7 +1229,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public AddCachePoolResponseProto addCachePool(RpcController controller,
AddCachePoolRequestProto request) throws ServiceException {
try {
- server.addCachePool(PBHelper.convert(request.getInfo()));
+ server.addCachePool(PBHelperClient.convert(request.getInfo()));
return AddCachePoolResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -1240,7 +1240,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public ModifyCachePoolResponseProto modifyCachePool(RpcController controller,
ModifyCachePoolRequestProto request) throws ServiceException {
try {
- server.modifyCachePool(PBHelper.convert(request.getInfo()));
+ server.modifyCachePool(PBHelperClient.convert(request.getInfo()));
return ModifyCachePoolResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -1268,7 +1268,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
ListCachePoolsResponseProto.newBuilder();
responseBuilder.setHasMore(entries.hasMore());
for (int i=0, n=entries.size(); i<n; i++) {
- responseBuilder.addEntries(PBHelper.convert(entries.get(i)));
+ responseBuilder.addEntries(PBHelperClient.convert(entries.get(i)));
}
return responseBuilder.build();
} catch (IOException e) {
@@ -1281,7 +1281,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, ModifyAclEntriesRequestProto req)
throws ServiceException {
try {
- server.modifyAclEntries(req.getSrc(), PBHelper.convertAclEntry(req.getAclSpecList()));
+ server.modifyAclEntries(req.getSrc(), PBHelperClient.convertAclEntry(req.getAclSpecList()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1294,7 +1294,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throws ServiceException {
try {
server.removeAclEntries(req.getSrc(),
- PBHelper.convertAclEntry(req.getAclSpecList()));
+ PBHelperClient.convertAclEntry(req.getAclSpecList()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1328,7 +1328,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public SetAclResponseProto setAcl(RpcController controller,
SetAclRequestProto req) throws ServiceException {
try {
- server.setAcl(req.getSrc(), PBHelper.convertAclEntry(req.getAclSpecList()));
+ server.setAcl(req.getSrc(), PBHelperClient.convertAclEntry(req.getAclSpecList()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1339,7 +1339,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public GetAclStatusResponseProto getAclStatus(RpcController controller,
GetAclStatusRequestProto req) throws ServiceException {
try {
- return PBHelper.convert(server.getAclStatus(req.getSrc()));
+ return PBHelperClient.convert(server.getAclStatus(req.getSrc()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1366,7 +1366,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
GetEZForPathResponseProto.newBuilder();
final EncryptionZone ret = server.getEZForPath(req.getSrc());
if (ret != null) {
- builder.setZone(PBHelper.convert(ret));
+ builder.setZone(PBHelperClient.convert(ret));
}
return builder.build();
} catch (IOException e) {
@@ -1385,7 +1385,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
ListEncryptionZonesResponseProto.newBuilder();
builder.setHasMore(entries.hasMore());
for (int i=0; i<entries.size(); i++) {
- builder.addZones(PBHelper.convert(entries.get(i)));
+ builder.addZones(PBHelperClient.convert(entries.get(i)));
}
return builder.build();
} catch (IOException e) {
@@ -1397,8 +1397,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public SetXAttrResponseProto setXAttr(RpcController controller,
SetXAttrRequestProto req) throws ServiceException {
try {
- server.setXAttr(req.getSrc(), PBHelper.convertXAttr(req.getXAttr()),
- PBHelper.convert(req.getFlag()));
+ server.setXAttr(req.getSrc(), PBHelperClient.convertXAttr(req.getXAttr()),
+ PBHelperClient.convert(req.getFlag()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1409,8 +1409,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public GetXAttrsResponseProto getXAttrs(RpcController controller,
GetXAttrsRequestProto req) throws ServiceException {
try {
- return PBHelper.convertXAttrsResponse(server.getXAttrs(req.getSrc(),
- PBHelper.convertXAttrs(req.getXAttrsList())));
+ return PBHelperClient.convertXAttrsResponse(server.getXAttrs(req.getSrc(),
+ PBHelperClient.convertXAttrs(req.getXAttrsList())));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1420,7 +1420,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public ListXAttrsResponseProto listXAttrs(RpcController controller,
ListXAttrsRequestProto req) throws ServiceException {
try {
- return PBHelper.convertListXAttrsResponse(server.listXAttrs(req.getSrc()));
+ return PBHelperClient.convertListXAttrsResponse(server.listXAttrs(req.getSrc()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1430,7 +1430,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public RemoveXAttrResponseProto removeXAttr(RpcController controller,
RemoveXAttrRequestProto req) throws ServiceException {
try {
- server.removeXAttr(req.getSrc(), PBHelper.convertXAttr(req.getXAttr()));
+ server.removeXAttr(req.getSrc(), PBHelperClient.convertXAttr(req.getXAttr()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1441,7 +1441,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public CheckAccessResponseProto checkAccess(RpcController controller,
CheckAccessRequestProto req) throws ServiceException {
try {
- server.checkAccess(req.getPath(), PBHelper.convert(req.getMode()));
+ server.checkAccess(req.getPath(), PBHelperClient.convert(req.getMode()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -1465,7 +1465,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, GetStoragePolicyRequestProto request)
throws ServiceException {
try {
- BlockStoragePolicyProto policy = PBHelper.convert(server
+ BlockStoragePolicyProto policy = PBHelperClient.convert(server
.getStoragePolicy(request.getPath()));
return GetStoragePolicyResponseProto.newBuilder()
.setStoragePolicy(policy).build();
@@ -1486,7 +1486,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return builder.build();
}
for (BlockStoragePolicy policy : policies) {
- builder.addPolicies(PBHelper.convert(policy));
+ builder.addPolicies(PBHelperClient.convert(policy));
}
return builder.build();
} catch (IOException e) {
@@ -1508,7 +1508,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public GetEditsFromTxidResponseProto getEditsFromTxid(RpcController controller,
GetEditsFromTxidRequestProto req) throws ServiceException {
try {
- return PBHelper.convertEditsResponse(server.getEditsFromTxid(
+ return PBHelperClient.convertEditsResponse(server.getEditsFromTxid(
req.getTxid()));
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index d30982a..f4ce46d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -255,8 +255,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
GetBlockLocationsResponseProto resp = rpcProxy.getBlockLocations(null,
req);
- return resp.hasLocations() ?
- PBHelper.convert(resp.getLocations()) : null;
+ return resp.hasLocations() ?
+ PBHelperClient.convert(resp.getLocations()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -266,7 +266,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
public FsServerDefaults getServerDefaults() throws IOException {
GetServerDefaultsRequestProto req = VOID_GET_SERVER_DEFAULT_REQUEST;
try {
- return PBHelper
+ return PBHelperClient
.convert(rpcProxy.getServerDefaults(null, req).getServerDefaults());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -285,17 +285,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
IOException {
CreateRequestProto.Builder builder = CreateRequestProto.newBuilder()
.setSrc(src)
- .setMasked(PBHelper.convert(masked))
+ .setMasked(PBHelperClient.convert(masked))
.setClientName(clientName)
- .setCreateFlag(PBHelper.convertCreateFlag(flag))
+ .setCreateFlag(PBHelperClient.convertCreateFlag(flag))
.setCreateParent(createParent)
.setReplication(replication)
.setBlockSize(blockSize);
- builder.addAllCryptoProtocolVersion(PBHelper.convert(supportedVersions));
+ builder.addAllCryptoProtocolVersion(PBHelperClient.convert(supportedVersions));
CreateRequestProto req = builder.build();
try {
CreateResponseProto res = rpcProxy.create(null, req);
- return res.hasFs() ? PBHelper.convert(res.getFs()) : null;
+ return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -323,13 +323,13 @@ public class ClientNamenodeProtocolTranslatorPB implements
DSQuotaExceededException, FileNotFoundException, SafeModeException,
UnresolvedLinkException, IOException {
AppendRequestProto req = AppendRequestProto.newBuilder().setSrc(src)
- .setClientName(clientName).setFlag(PBHelper.convertCreateFlag(flag))
+ .setClientName(clientName).setFlag(PBHelperClient.convertCreateFlag(flag))
.build();
try {
AppendResponseProto res = rpcProxy.append(null, req);
- LocatedBlock lastBlock = res.hasBlock() ? PBHelper
+ LocatedBlock lastBlock = res.hasBlock() ? PBHelperClient
.convert(res.getBlock()) : null;
- HdfsFileStatus stat = (res.hasStat()) ? PBHelper.convert(res.getStat())
+ HdfsFileStatus stat = (res.hasStat()) ? PBHelperClient.convert(res.getStat())
: null;
return new LastBlockWithStatus(lastBlock, stat);
} catch (ServiceException e) {
@@ -359,7 +359,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
UnresolvedLinkException, IOException {
SetPermissionRequestProto req = SetPermissionRequestProto.newBuilder()
.setSrc(src)
- .setPermission(PBHelper.convert(permission))
+ .setPermission(PBHelperClient.convert(permission))
.build();
try {
rpcProxy.setPermission(null, req);
@@ -416,7 +416,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
req.addAllFavoredNodes(Arrays.asList(favoredNodes));
}
try {
- return PBHelper.convert(rpcProxy.addBlock(null, req.build()).getBlock());
+ return PBHelperClient.convert(rpcProxy.addBlock(null, req.build()).getBlock());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -441,7 +441,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setClientName(clientName)
.build();
try {
- return PBHelper.convert(rpcProxy.getAdditionalDatanode(null, req)
+ return PBHelperClient.convert(rpcProxy.getAdditionalDatanode(null, req)
.getBlock());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -469,7 +469,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
ReportBadBlocksRequestProto req = ReportBadBlocksRequestProto.newBuilder()
- .addAllBlocks(Arrays.asList(PBHelper.convertLocatedBlock(blocks)))
+ .addAllBlocks(Arrays.asList(PBHelperClient.convertLocatedBlock(blocks)))
.build();
try {
rpcProxy.reportBadBlocks(null, req);
@@ -552,7 +552,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
IOException {
MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
.setSrc(src)
- .setMasked(PBHelper.convert(masked))
+ .setMasked(PBHelperClient.convert(masked))
.setCreateParent(createParent).build();
try {
@@ -574,7 +574,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetListingResponseProto result = rpcProxy.getListing(null, req);
if (result.hasDirList()) {
- return PBHelper.convert(result.getDirList());
+ return PBHelperClient.convert(result.getDirList());
}
return null;
} catch (ServiceException e) {
@@ -610,7 +610,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public long[] getStats() throws IOException {
try {
- return PBHelper.convert(rpcProxy.getFsStats(null,
+ return PBHelperClient.convert(rpcProxy.getFsStats(null,
VOID_GET_FSSTATUS_REQUEST));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -622,9 +622,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
throws IOException {
GetDatanodeReportRequestProto req = GetDatanodeReportRequestProto
.newBuilder()
- .setType(PBHelper.convert(type)).build();
+ .setType(PBHelperClient.convert(type)).build();
try {
- return PBHelper.convert(
+ return PBHelperClient.convert(
rpcProxy.getDatanodeReport(null, req).getDiList());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -636,9 +636,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
throws IOException {
final GetDatanodeStorageReportRequestProto req
= GetDatanodeStorageReportRequestProto.newBuilder()
- .setType(PBHelper.convert(type)).build();
+ .setType(PBHelperClient.convert(type)).build();
try {
- return PBHelper.convertDatanodeStorageReports(
+ return PBHelperClient.convertDatanodeStorageReports(
rpcProxy.getDatanodeStorageReport(null, req).getDatanodeStorageReportsList());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -662,7 +662,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException {
SetSafeModeRequestProto req = SetSafeModeRequestProto.newBuilder()
- .setAction(PBHelper.convert(action)).setChecked(isChecked).build();
+ .setAction(PBHelperClient.convert(action)).setChecked(isChecked).build();
try {
return rpcProxy.setSafeMode(null, req).getResult();
} catch (ServiceException e) {
@@ -726,11 +726,11 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException {
final RollingUpgradeRequestProto r = RollingUpgradeRequestProto.newBuilder()
- .setAction(PBHelper.convert(action)).build();
+ .setAction(PBHelperClient.convert(action)).build();
try {
final RollingUpgradeResponseProto proto = rpcProxy.rollingUpgrade(null, r);
if (proto.hasRollingUpgradeInfo()) {
- return PBHelper.convert(proto.getRollingUpgradeInfo());
+ return PBHelperClient.convert(proto.getRollingUpgradeInfo());
}
return null;
} catch (ServiceException e) {
@@ -746,7 +746,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
if (cookie != null)
req.setCookie(cookie);
try {
- return PBHelper.convert(
+ return PBHelperClient.convert(
rpcProxy.listCorruptFileBlocks(null, req.build()).getCorrupt());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -772,7 +772,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setSrc(src).build();
try {
GetFileInfoResponseProto res = rpcProxy.getFileInfo(null, req);
- return res.hasFs() ? PBHelper.convert(res.getFs()) : null;
+ return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -786,7 +786,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
GetFileLinkInfoResponseProto result = rpcProxy.getFileLinkInfo(null, req);
return result.hasFs() ?
- PBHelper.convert(rpcProxy.getFileLinkInfo(null, req).getFs()) : null;
+ PBHelperClient.convert(rpcProxy.getFileLinkInfo(null, req).getFs()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -801,7 +801,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setPath(path)
.build();
try {
- return PBHelper.convert(rpcProxy.getContentSummary(null, req)
+ return PBHelperClient.convert(rpcProxy.getContentSummary(null, req)
.getSummary());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -869,7 +869,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
CreateSymlinkRequestProto req = CreateSymlinkRequestProto.newBuilder()
.setTarget(target)
.setLink(link)
- .setDirPerm(PBHelper.convert(dirPerm))
+ .setDirPerm(PBHelperClient.convert(dirPerm))
.setCreateParent(createParent)
.build();
try {
@@ -901,7 +901,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setClientName(clientName)
.build();
try {
- return PBHelper.convert(
+ return PBHelperClient.convert(
rpcProxy.updateBlockForPipeline(null, req).getBlock());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -915,7 +915,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setClientName(clientName)
.setOldBlock(PBHelperClient.convert(oldBlock))
.setNewBlock(PBHelperClient.convert(newBlock))
- .addAllNewNodes(Arrays.asList(PBHelper.convert(newNodes)))
+ .addAllNewNodes(Arrays.asList(PBHelperClient.convert(newNodes)))
.addAllStorageIDs(storageIDs == null ? null : Arrays.asList(storageIDs))
.build();
try {
@@ -934,7 +934,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.build();
try {
GetDelegationTokenResponseProto resp = rpcProxy.getDelegationToken(null, req);
- return resp.hasToken() ? PBHelper.convertDelegationToken(resp.getToken())
+ return resp.hasToken() ? PBHelperClient.convertDelegationToken(resp.getToken())
: null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -993,7 +993,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetDataEncryptionKeyResponseProto rsp = rpcProxy.getDataEncryptionKey(
null, VOID_GET_DATA_ENCRYPTIONKEY_REQUEST);
return rsp.hasDataEncryptionKey() ?
- PBHelper.convert(rsp.getDataEncryptionKey()) : null;
+ PBHelperClient.convert(rsp.getDataEncryptionKey()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1090,7 +1090,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.getSnapshottableDirListing(null, req);
if (result.hasSnapshottableDirList()) {
- return PBHelper.convert(result.getSnapshottableDirList());
+ return PBHelperClient.convert(result.getSnapshottableDirList());
}
return null;
} catch (ServiceException e) {
@@ -1108,7 +1108,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetSnapshotDiffReportResponseProto result =
rpcProxy.getSnapshotDiffReport(null, req);
- return PBHelper.convert(result.getDiffReport());
+ return PBHelperClient.convert(result.getDiffReport());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1120,9 +1120,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
AddCacheDirectiveRequestProto.Builder builder =
AddCacheDirectiveRequestProto.newBuilder().
- setInfo(PBHelper.convert(directive));
+ setInfo(PBHelperClient.convert(directive));
if (!flags.isEmpty()) {
- builder.setCacheFlags(PBHelper.convertCacheFlags(flags));
+ builder.setCacheFlags(PBHelperClient.convertCacheFlags(flags));
}
return rpcProxy.addCacheDirective(null, builder.build()).getId();
} catch (ServiceException e) {
@@ -1136,9 +1136,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
ModifyCacheDirectiveRequestProto.Builder builder =
ModifyCacheDirectiveRequestProto.newBuilder().
- setInfo(PBHelper.convert(directive));
+ setInfo(PBHelperClient.convert(directive));
if (!flags.isEmpty()) {
- builder.setCacheFlags(PBHelper.convertCacheFlags(flags));
+ builder.setCacheFlags(PBHelperClient.convertCacheFlags(flags));
}
rpcProxy.modifyCacheDirective(null, builder.build());
} catch (ServiceException e) {
@@ -1169,7 +1169,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public CacheDirectiveEntry get(int i) {
- return PBHelper.convert(response.getElements(i));
+ return PBHelperClient.convert(response.getElements(i));
}
@Override
@@ -1195,7 +1195,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
rpcProxy.listCacheDirectives(null,
ListCacheDirectivesRequestProto.newBuilder().
setPrevId(prevId).
- setFilter(PBHelper.convert(filter)).
+ setFilter(PBHelperClient.convert(filter)).
build()));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -1206,7 +1206,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
public void addCachePool(CachePoolInfo info) throws IOException {
AddCachePoolRequestProto.Builder builder =
AddCachePoolRequestProto.newBuilder();
- builder.setInfo(PBHelper.convert(info));
+ builder.setInfo(PBHelperClient.convert(info));
try {
rpcProxy.addCachePool(null, builder.build());
} catch (ServiceException e) {
@@ -1218,7 +1218,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
public void modifyCachePool(CachePoolInfo req) throws IOException {
ModifyCachePoolRequestProto.Builder builder =
ModifyCachePoolRequestProto.newBuilder();
- builder.setInfo(PBHelper.convert(req));
+ builder.setInfo(PBHelperClient.convert(req));
try {
rpcProxy.modifyCachePool(null, builder.build());
} catch (ServiceException e) {
@@ -1248,7 +1248,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public CachePoolEntry get(int i) {
CachePoolEntryProto elem = proto.getEntries(i);
- return PBHelper.convert(elem);
+ return PBHelperClient.convert(elem);
}
@Override
@@ -1280,7 +1280,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
throws IOException {
ModifyAclEntriesRequestProto req = ModifyAclEntriesRequestProto
.newBuilder().setSrc(src)
- .addAllAclSpec(PBHelper.convertAclEntryProto(aclSpec)).build();
+ .addAllAclSpec(PBHelperClient.convertAclEntryProto(aclSpec)).build();
try {
rpcProxy.modifyAclEntries(null, req);
} catch (ServiceException e) {
@@ -1293,7 +1293,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
throws IOException {
RemoveAclEntriesRequestProto req = RemoveAclEntriesRequestProto
.newBuilder().setSrc(src)
- .addAllAclSpec(PBHelper.convertAclEntryProto(aclSpec)).build();
+ .addAllAclSpec(PBHelperClient.convertAclEntryProto(aclSpec)).build();
try {
rpcProxy.removeAclEntries(null, req);
} catch (ServiceException e) {
@@ -1327,7 +1327,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
SetAclRequestProto req = SetAclRequestProto.newBuilder()
.setSrc(src)
- .addAllAclSpec(PBHelper.convertAclEntryProto(aclSpec))
+ .addAllAclSpec(PBHelperClient.convertAclEntryProto(aclSpec))
.build();
try {
rpcProxy.setAcl(null, req);
@@ -1341,7 +1341,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetAclStatusRequestProto req = GetAclStatusRequestProto.newBuilder()
.setSrc(src).build();
try {
- return PBHelper.convert(rpcProxy.getAclStatus(null, req));
+ return PBHelperClient.convert(rpcProxy.getAclStatus(null, req));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1375,7 +1375,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
final EncryptionZonesProtos.GetEZForPathResponseProto response =
rpcProxy.getEZForPath(null, req);
if (response.hasZone()) {
- return PBHelper.convert(response.getZone());
+ return PBHelperClient.convert(response.getZone());
} else {
return null;
}
@@ -1397,7 +1397,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
List<EncryptionZone> elements =
Lists.newArrayListWithCapacity(response.getZonesCount());
for (EncryptionZoneProto p : response.getZonesList()) {
- elements.add(PBHelper.convert(p));
+ elements.add(PBHelperClient.convert(p));
}
return new BatchedListEntries<EncryptionZone>(elements,
response.getHasMore());
@@ -1411,8 +1411,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
throws IOException {
SetXAttrRequestProto req = SetXAttrRequestProto.newBuilder()
.setSrc(src)
- .setXAttr(PBHelper.convertXAttrProto(xAttr))
- .setFlag(PBHelper.convert(flag))
+ .setXAttr(PBHelperClient.convertXAttrProto(xAttr))
+ .setFlag(PBHelperClient.convert(flag))
.build();
try {
rpcProxy.setXAttr(null, req);
@@ -1422,16 +1422,16 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
- public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
+ public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
throws IOException {
GetXAttrsRequestProto.Builder builder = GetXAttrsRequestProto.newBuilder();
builder.setSrc(src);
if (xAttrs != null) {
- builder.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ builder.addAllXAttrs(PBHelperClient.convertXAttrProto(xAttrs));
}
GetXAttrsRequestProto req = builder.build();
try {
- return PBHelper.convert(rpcProxy.getXAttrs(null, req));
+ return PBHelperClient.convert(rpcProxy.getXAttrs(null, req));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1444,7 +1444,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
builder.setSrc(src);
ListXAttrsRequestProto req = builder.build();
try {
- return PBHelper.convert(rpcProxy.listXAttrs(null, req));
+ return PBHelperClient.convert(rpcProxy.listXAttrs(null, req));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1454,7 +1454,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
public void removeXAttr(String src, XAttr xAttr) throws IOException {
RemoveXAttrRequestProto req = RemoveXAttrRequestProto
.newBuilder().setSrc(src)
- .setXAttr(PBHelper.convertXAttrProto(xAttr)).build();
+ .setXAttr(PBHelperClient.convertXAttrProto(xAttr)).build();
try {
rpcProxy.removeXAttr(null, req);
} catch (ServiceException e) {
@@ -1465,7 +1465,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public void checkAccess(String path, FsAction mode) throws IOException {
CheckAccessRequestProto req = CheckAccessRequestProto.newBuilder()
- .setPath(path).setMode(PBHelper.convert(mode)).build();
+ .setPath(path).setMode(PBHelperClient.convert(mode)).build();
try {
rpcProxy.checkAccess(null, req);
} catch (ServiceException e) {
@@ -1490,7 +1490,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetStoragePolicyRequestProto request = GetStoragePolicyRequestProto
.newBuilder().setPath(path).build();
try {
- return PBHelper.convert(rpcProxy.getStoragePolicy(null, request)
+ return PBHelperClient.convert(rpcProxy.getStoragePolicy(null, request)
.getStoragePolicy());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -1502,7 +1502,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
GetStoragePoliciesResponseProto response = rpcProxy
.getStoragePolicies(null, VOID_GET_STORAGE_POLICIES_REQUEST);
- return PBHelper.convertStoragePolicies(response.getPoliciesList());
+ return PBHelperClient.convertStoragePolicies(response.getPoliciesList());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1523,7 +1523,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
.setTxid(txid).build();
try {
- return PBHelper.convert(rpcProxy.getEditsFromTxid(null, req));
+ return PBHelperClient.convert(rpcProxy.getEditsFromTxid(null, req));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
index 0b46927..18f89f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
@@ -139,7 +139,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
.setXmitsInProgress(xmitsInProgress).setXceiverCount(xceiverCount)
.setFailedVolumes(failedVolumes)
.setRequestFullBlockReportLease(requestFullBlockReportLease);
- builder.addAllReports(PBHelper.convertStorageReports(reports));
+ builder.addAllReports(PBHelperClient.convertStorageReports(reports));
if (cacheCapacity != 0) {
builder.setCacheCapacity(cacheCapacity);
}
@@ -164,7 +164,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
}
RollingUpgradeStatus rollingUpdateStatus = null;
if (resp.hasRollingUpgradeStatus()) {
- rollingUpdateStatus = PBHelper.convert(resp.getRollingUpgradeStatus());
+ rollingUpdateStatus = PBHelperClient.convert(resp.getRollingUpgradeStatus());
}
return new HeartbeatResponse(cmds, PBHelper.convert(resp.getHaStatus()),
rollingUpdateStatus, resp.getFullBlockReportLeaseId());
@@ -183,7 +183,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
for (StorageBlockReport r : reports) {
StorageBlockReportProto.Builder reportBuilder = StorageBlockReportProto
- .newBuilder().setStorage(PBHelper.convert(r.getStorage()));
+ .newBuilder().setStorage(PBHelperClient.convert(r.getStorage()));
BlockListAsLongs blocks = r.getBlocks();
if (useBlocksBuffer) {
reportBuilder.setNumberOfBlocks(blocks.getNumberOfBlocks());
@@ -240,7 +240,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
StorageReceivedDeletedBlocksProto.Builder repBuilder =
StorageReceivedDeletedBlocksProto.newBuilder();
repBuilder.setStorageUuid(storageBlock.getStorage().getStorageID()); // Set for wire compatibility.
- repBuilder.setStorage(PBHelper.convert(storageBlock.getStorage()));
+ repBuilder.setStorage(PBHelperClient.convert(storageBlock.getStorage()));
for (ReceivedDeletedBlockInfo rdBlock : storageBlock.getBlocks()) {
repBuilder.addBlocks(PBHelper.convert(rdBlock));
}
@@ -281,7 +281,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
ReportBadBlocksRequestProto.Builder builder = ReportBadBlocksRequestProto
.newBuilder();
for (int i = 0; i < blocks.length; i++) {
- builder.addBlocks(i, PBHelper.convert(blocks[i]));
+ builder.addBlocks(i, PBHelperClient.convert(blocks[i]));
}
ReportBadBlocksRequestProto req = builder.build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
index 5964e15..94d1f0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
@@ -105,7 +105,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
HeartbeatRequestProto request) throws ServiceException {
HeartbeatResponse response;
try {
- final StorageReport[] report = PBHelper.convertStorageReports(
+ final StorageReport[] report = PBHelperClient.convertStorageReports(
request.getReportsList());
VolumeFailureSummary volumeFailureSummary =
request.hasVolumeFailureSummary() ? PBHelper.convertVolumeFailureSummary(
@@ -132,7 +132,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
RollingUpgradeStatus rollingUpdateStatus = response
.getRollingUpdateStatus();
if (rollingUpdateStatus != null) {
- builder.setRollingUpgradeStatus(PBHelper
+ builder.setRollingUpgradeStatus(PBHelperClient
.convertRollingUpgradeStatus(rollingUpdateStatus));
}
builder.setFullBlockReportLeaseId(response.getFullBlockReportLeaseId());
@@ -157,7 +157,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
} else {
blocks = BlockListAsLongs.decodeLongs(s.getBlocksList());
}
- report[index++] = new StorageBlockReport(PBHelper.convert(s.getStorage()),
+ report[index++] = new StorageBlockReport(PBHelperClient.convert(s.getStorage()),
blocks);
}
try {
@@ -214,7 +214,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
}
if (sBlock.hasStorage()) {
info[i] = new StorageReceivedDeletedBlocks(
- PBHelper.convert(sBlock.getStorage()), rdBlocks);
+ PBHelperClient.convert(sBlock.getStorage()), rdBlocks);
} else {
info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageUuid(), rdBlocks);
}
@@ -259,7 +259,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
List<LocatedBlockProto> lbps = request.getBlocksList();
LocatedBlock [] blocks = new LocatedBlock [lbps.size()];
for(int i=0; i<lbps.size(); i++) {
- blocks[i] = PBHelper.convert(lbps.get(i));
+ blocks[i] = PBHelperClient.convert(lbps.get(i));
}
try {
impl.reportBadBlocks(blocks);
@@ -276,7 +276,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
List<DatanodeIDProto> dnprotos = request.getNewTaragetsList();
DatanodeID[] dns = new DatanodeID[dnprotos.size()];
for (int i = 0; i < dnprotos.size(); i++) {
- dns[i] = PBHelper.convert(dnprotos.get(i));
+ dns[i] = PBHelperClient.convert(dnprotos.get(i));
}
final List<String> sidprotos = request.getNewTargetStoragesList();
final String[] storageIDs = sidprotos.toArray(new String[sidprotos.size()]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
index fb67e36..ff4277c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
@@ -64,7 +64,7 @@ public class InterDatanodeProtocolServerSideTranslatorPB implements
} else {
return InitReplicaRecoveryResponseProto.newBuilder()
.setReplicaFound(true)
- .setBlock(PBHelper.convert(r))
+ .setBlock(PBHelperClient.convert(r))
.setState(PBHelper.convert(r.getOriginalReplicaState())).build();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
index 49fdf5d..44e814e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
@@ -67,7 +67,7 @@ public class JournalProtocolTranslatorPB implements ProtocolMetaInterface,
.setEpoch(epoch)
.setFirstTxnId(firstTxnId)
.setNumTxns(numTxns)
- .setRecords(PBHelper.getByteString(records))
+ .setRecords(PBHelperClient.getByteString(records))
.build();
try {
rpcProxy.journal(NULL_CONTROLLER, req);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index dbacded..91ffb1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -79,7 +78,7 @@ public class NamenodeProtocolServerSideTranslatorPB implements
@Override
public GetBlocksResponseProto getBlocks(RpcController unused,
GetBlocksRequestProto request) throws ServiceException {
- DatanodeInfo dnInfo = new DatanodeInfo(PBHelper.convert(request
+ DatanodeInfo dnInfo = new DatanodeInfo(PBHelperClient.convert(request
.getDatanode()));
BlocksWithLocations blocks;
try {
[12/32] hadoop git commit: HDFS-9111. Move hdfs-client protobuf
convert methods from PBHelper to PBHelperClient. Contributed by Mingliang
Liu.
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index cf55445..3de4513 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -17,97 +17,21 @@
*/
package org.apache.hadoop.hdfs.protocolPB;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos
- .EncryptionZoneProto;
-import static org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CryptoProtocolVersionProto;
-
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.EnumSet;
import java.util.List;
-import org.apache.hadoop.fs.CacheFlag;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FsServerDefaults;
-import org.apache.hadoop.fs.Path;
+import com.google.protobuf.ByteString;
+
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.fs.XAttr;
-import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.permission.AclEntry;
-import org.apache.hadoop.fs.permission.AclEntryScope;
-import org.apache.hadoop.fs.permission.AclEntryType;
-import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.inotify.EventBatch;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
-import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
-import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
-import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
-import org.apache.hadoop.hdfs.protocol.CachePoolStats;
-import org.apache.hadoop.crypto.CipherOption;
-import org.apache.hadoop.crypto.CipherSuite;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
-import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
-import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
-import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.fs.FileEncryptionInfo;
-import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
-import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
-import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
-import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto;
-import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto;
-import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto;
-import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto;
-import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto;
-import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveStatsProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheFlagProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolEntryProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolStatsProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeStorageReportProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeActionProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockIdCommandProto;
@@ -121,34 +45,15 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDele
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.VolumeFailureSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockReportContextProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CipherOptionProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DataEncryptionKeyProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto.AdminState;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfosProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeLocalInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.StorageState;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto;
@@ -157,28 +62,13 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RecoveringBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogManifestProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReplicaStateProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RollingUpgradeStatusProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportEntryProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
-import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto;
-import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto;
-import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrSetFlagProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -196,9 +86,6 @@ import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
@@ -211,18 +98,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStat
import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
-import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
-import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
-import org.apache.hadoop.io.EnumSetWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
-import org.apache.hadoop.security.token.Token;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Shorts;
-import com.google.protobuf.ByteString;
/**
* Utilities for converting protobuf classes to and from implementation classes
@@ -231,33 +107,19 @@ import com.google.protobuf.ByteString;
* Note that when converting from an internal type to protobuf type, the
* converter never return null for protobuf type. The check for internal type
* being null must be done before calling the convert() method.
+ *
+ * For those helper methods that convert HDFS client-side data structures from
+ * and to protobuf, see {@link PBHelperClient}.
*/
public class PBHelper {
private static final RegisterCommandProto REG_CMD_PROTO =
RegisterCommandProto.newBuilder().build();
private static final RegisterCommand REG_CMD = new RegisterCommand();
- private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES =
- AclEntryScope.values();
- private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES =
- AclEntryType.values();
- private static final FsAction[] FSACTION_VALUES =
- FsAction.values();
- private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
- XAttr.NameSpace.values();
-
private PBHelper() {
/** Hidden constructor */
}
- public static ByteString getByteString(byte[] bytes) {
- return ByteString.copyFrom(bytes);
- }
-
- private static <T extends Enum<T>, U extends Enum<U>> U castEnum(T from, U[] to) {
- return to[from.ordinal()];
- }
-
public static NamenodeRole convert(NamenodeRoleProto role) {
switch (role) {
case NAMENODE:
@@ -282,65 +144,6 @@ public class PBHelper {
return null;
}
- public static BlockStoragePolicy[] convertStoragePolicies(
- List<BlockStoragePolicyProto> policyProtos) {
- if (policyProtos == null || policyProtos.size() == 0) {
- return new BlockStoragePolicy[0];
- }
- BlockStoragePolicy[] policies = new BlockStoragePolicy[policyProtos.size()];
- int i = 0;
- for (BlockStoragePolicyProto proto : policyProtos) {
- policies[i++] = convert(proto);
- }
- return policies;
- }
-
- public static BlockStoragePolicy convert(BlockStoragePolicyProto proto) {
- List<StorageTypeProto> cList = proto.getCreationPolicy()
- .getStorageTypesList();
- StorageType[] creationTypes = convertStorageTypes(cList, cList.size());
- List<StorageTypeProto> cfList = proto.hasCreationFallbackPolicy() ? proto
- .getCreationFallbackPolicy().getStorageTypesList() : null;
- StorageType[] creationFallbackTypes = cfList == null ? StorageType
- .EMPTY_ARRAY : convertStorageTypes(cfList, cfList.size());
- List<StorageTypeProto> rfList = proto.hasReplicationFallbackPolicy() ?
- proto.getReplicationFallbackPolicy().getStorageTypesList() : null;
- StorageType[] replicationFallbackTypes = rfList == null ? StorageType
- .EMPTY_ARRAY : convertStorageTypes(rfList, rfList.size());
- return new BlockStoragePolicy((byte) proto.getPolicyId(), proto.getName(),
- creationTypes, creationFallbackTypes, replicationFallbackTypes);
- }
-
- public static BlockStoragePolicyProto convert(BlockStoragePolicy policy) {
- BlockStoragePolicyProto.Builder builder = BlockStoragePolicyProto
- .newBuilder().setPolicyId(policy.getId()).setName(policy.getName());
- // creation storage types
- StorageTypesProto creationProto = convert(policy.getStorageTypes());
- Preconditions.checkArgument(creationProto != null);
- builder.setCreationPolicy(creationProto);
- // creation fallback
- StorageTypesProto creationFallbackProto = convert(
- policy.getCreationFallbacks());
- if (creationFallbackProto != null) {
- builder.setCreationFallbackPolicy(creationFallbackProto);
- }
- // replication fallback
- StorageTypesProto replicationFallbackProto = convert(
- policy.getReplicationFallbacks());
- if (replicationFallbackProto != null) {
- builder.setReplicationFallbackPolicy(replicationFallbackProto);
- }
- return builder.build();
- }
-
- public static StorageTypesProto convert(StorageType[] types) {
- if (types == null || types.length == 0) {
- return null;
- }
- List<StorageTypeProto> list = PBHelperClient.convertStorageTypes(types);
- return StorageTypesProto.newBuilder().addAllStorageTypes(list).build();
- }
-
public static StorageInfoProto convert(StorageInfo info) {
return StorageInfoProto.newBuilder().setClusterID(info.getClusterID())
.setCTime(info.getCTime()).setLayoutVersion(info.getLayoutVersion())
@@ -365,49 +168,9 @@ public class PBHelper {
si, convert(reg.getRole()));
}
- // DatanodeId
- public static DatanodeID convert(DatanodeIDProto dn) {
- return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
- dn.getXferPort(), dn.getInfoPort(), dn.hasInfoSecurePort() ? dn
- .getInfoSecurePort() : 0, dn.getIpcPort());
- }
-
- // Arrays of DatanodeId
- public static DatanodeIDProto[] convert(DatanodeID[] did) {
- if (did == null)
- return null;
- final int len = did.length;
- DatanodeIDProto[] result = new DatanodeIDProto[len];
- for (int i = 0; i < len; ++i) {
- result[i] = PBHelperClient.convert(did[i]);
- }
- return result;
- }
-
- public static DatanodeID[] convert(DatanodeIDProto[] did) {
- if (did == null) return null;
- final int len = did.length;
- DatanodeID[] result = new DatanodeID[len];
- for (int i = 0; i < len; ++i) {
- result[i] = convert(did[i]);
- }
- return result;
- }
-
- // Block
- public static BlockProto convert(Block b) {
- return BlockProto.newBuilder().setBlockId(b.getBlockId())
- .setGenStamp(b.getGenerationStamp()).setNumBytes(b.getNumBytes())
- .build();
- }
-
- public static Block convert(BlockProto b) {
- return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
- }
-
public static BlockWithLocationsProto convert(BlockWithLocations blk) {
return BlockWithLocationsProto.newBuilder()
- .setBlock(convert(blk.getBlock()))
+ .setBlock(PBHelperClient.convert(blk.getBlock()))
.addAllDatanodeUuids(Arrays.asList(blk.getDatanodeUuids()))
.addAllStorageUuids(Arrays.asList(blk.getStorageIDs()))
.addAllStorageTypes(PBHelperClient.convertStorageTypes(blk.getStorageTypes()))
@@ -418,10 +181,10 @@ public class PBHelper {
final List<String> datanodeUuids = b.getDatanodeUuidsList();
final List<String> storageUuids = b.getStorageUuidsList();
final List<StorageTypeProto> storageTypes = b.getStorageTypesList();
- return new BlockWithLocations(convert(b.getBlock()),
+ return new BlockWithLocations(PBHelperClient.convert(b.getBlock()),
datanodeUuids.toArray(new String[datanodeUuids.size()]),
storageUuids.toArray(new String[storageUuids.size()]),
- convertStorageTypes(storageTypes, storageUuids.size()));
+ PBHelperClient.convertStorageTypes(storageTypes, storageUuids.size()));
}
public static BlocksWithLocationsProto convert(BlocksWithLocations blks) {
@@ -445,7 +208,7 @@ public class PBHelper {
public static BlockKeyProto convert(BlockKey key) {
byte[] encodedKey = key.getEncodedKey();
- ByteString keyBytes = ByteString.copyFrom(encodedKey == null ?
+ ByteString keyBytes = ByteString.copyFrom(encodedKey == null ?
DFSUtilClient.EMPTY_BYTES : encodedKey);
return BlockKeyProto.newBuilder().setKeyId(key.getKeyId())
.setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build();
@@ -570,185 +333,22 @@ public class PBHelper {
if (b == null) {
return null;
}
- LocatedBlockProto lb = PBHelper.convert((LocatedBlock)b);
+ LocatedBlockProto lb = PBHelperClient.convert((LocatedBlock) b);
RecoveringBlockProto.Builder builder = RecoveringBlockProto.newBuilder();
builder.setBlock(lb).setNewGenStamp(b.getNewGenerationStamp());
if(b.getNewBlock() != null)
- builder.setTruncateBlock(PBHelper.convert(b.getNewBlock()));
+ builder.setTruncateBlock(PBHelperClient.convert(b.getNewBlock()));
return builder.build();
}
public static RecoveringBlock convert(RecoveringBlockProto b) {
ExtendedBlock block = PBHelperClient.convert(b.getBlock().getB());
- DatanodeInfo[] locs = convert(b.getBlock().getLocsList());
+ DatanodeInfo[] locs = PBHelperClient.convert(b.getBlock().getLocsList());
return (b.hasTruncateBlock()) ?
- new RecoveringBlock(block, locs, PBHelper.convert(b.getTruncateBlock())) :
+ new RecoveringBlock(block, locs, PBHelperClient.convert(b.getTruncateBlock())) :
new RecoveringBlock(block, locs, b.getNewGenStamp());
}
- static public DatanodeInfo convert(DatanodeInfoProto di) {
- if (di == null) return null;
- return new DatanodeInfo(
- PBHelper.convert(di.getId()),
- di.hasLocation() ? di.getLocation() : null ,
- di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
- di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
- di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), PBHelper.convert(di.getAdminState()),
- di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
- }
-
- static public DatanodeInfo[] convert(DatanodeInfoProto di[]) {
- if (di == null) return null;
- DatanodeInfo[] result = new DatanodeInfo[di.length];
- for (int i = 0; i < di.length; i++) {
- result[i] = convert(di[i]);
- }
- return result;
- }
-
- public static DatanodeInfo[] convert(List<DatanodeInfoProto> list) {
- DatanodeInfo[] info = new DatanodeInfo[list.size()];
- for (int i = 0; i < info.length; i++) {
- info[i] = convert(list.get(i));
- }
- return info;
- }
-
- public static DatanodeStorageReportProto convertDatanodeStorageReport(
- DatanodeStorageReport report) {
- return DatanodeStorageReportProto.newBuilder()
- .setDatanodeInfo(PBHelperClient.convert(report.getDatanodeInfo()))
- .addAllStorageReports(convertStorageReports(report.getStorageReports()))
- .build();
- }
-
- public static List<DatanodeStorageReportProto> convertDatanodeStorageReports(
- DatanodeStorageReport[] reports) {
- final List<DatanodeStorageReportProto> protos
- = new ArrayList<DatanodeStorageReportProto>(reports.length);
- for(int i = 0; i < reports.length; i++) {
- protos.add(convertDatanodeStorageReport(reports[i]));
- }
- return protos;
- }
-
- public static DatanodeStorageReport convertDatanodeStorageReport(
- DatanodeStorageReportProto proto) {
- return new DatanodeStorageReport(
- convert(proto.getDatanodeInfo()),
- convertStorageReports(proto.getStorageReportsList()));
- }
-
- public static DatanodeStorageReport[] convertDatanodeStorageReports(
- List<DatanodeStorageReportProto> protos) {
- final DatanodeStorageReport[] reports
- = new DatanodeStorageReport[protos.size()];
- for(int i = 0; i < reports.length; i++) {
- reports[i] = convertDatanodeStorageReport(protos.get(i));
- }
- return reports;
- }
-
- public static AdminStates convert(AdminState adminState) {
- switch(adminState) {
- case DECOMMISSION_INPROGRESS:
- return AdminStates.DECOMMISSION_INPROGRESS;
- case DECOMMISSIONED:
- return AdminStates.DECOMMISSIONED;
- case NORMAL:
- default:
- return AdminStates.NORMAL;
- }
- }
-
- public static LocatedBlockProto convert(LocatedBlock b) {
- if (b == null) return null;
- Builder builder = LocatedBlockProto.newBuilder();
- DatanodeInfo[] locs = b.getLocations();
- List<DatanodeInfo> cachedLocs =
- Lists.newLinkedList(Arrays.asList(b.getCachedLocations()));
- for (int i = 0; i < locs.length; i++) {
- DatanodeInfo loc = locs[i];
- builder.addLocs(i, PBHelperClient.convert(loc));
- boolean locIsCached = cachedLocs.contains(loc);
- builder.addIsCached(locIsCached);
- if (locIsCached) {
- cachedLocs.remove(loc);
- }
- }
- Preconditions.checkArgument(cachedLocs.size() == 0,
- "Found additional cached replica locations that are not in the set of"
- + " storage-backed locations!");
-
- StorageType[] storageTypes = b.getStorageTypes();
- if (storageTypes != null) {
- for (int i = 0; i < storageTypes.length; ++i) {
- builder.addStorageTypes(PBHelperClient.convertStorageType(storageTypes[i]));
- }
- }
- final String[] storageIDs = b.getStorageIDs();
- if (storageIDs != null) {
- builder.addAllStorageIDs(Arrays.asList(storageIDs));
- }
-
- return builder.setB(PBHelperClient.convert(b.getBlock()))
- .setBlockToken(PBHelperClient.convert(b.getBlockToken()))
- .setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build();
- }
-
- public static LocatedBlock convert(LocatedBlockProto proto) {
- if (proto == null) return null;
- List<DatanodeInfoProto> locs = proto.getLocsList();
- DatanodeInfo[] targets = new DatanodeInfo[locs.size()];
- for (int i = 0; i < locs.size(); i++) {
- targets[i] = PBHelper.convert(locs.get(i));
- }
-
- final StorageType[] storageTypes = convertStorageTypes(
- proto.getStorageTypesList(), locs.size());
-
- final int storageIDsCount = proto.getStorageIDsCount();
- final String[] storageIDs;
- if (storageIDsCount == 0) {
- storageIDs = null;
- } else {
- Preconditions.checkState(storageIDsCount == locs.size());
- storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
- }
-
- // Set values from the isCached list, re-using references from loc
- List<DatanodeInfo> cachedLocs = new ArrayList<DatanodeInfo>(locs.size());
- List<Boolean> isCachedList = proto.getIsCachedList();
- for (int i=0; i<isCachedList.size(); i++) {
- if (isCachedList.get(i)) {
- cachedLocs.add(targets[i]);
- }
- }
-
- LocatedBlock lb = new LocatedBlock(PBHelperClient.convert(proto.getB()), targets,
- storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
- cachedLocs.toArray(new DatanodeInfo[0]));
- lb.setBlockToken(PBHelper.convert(proto.getBlockToken()));
-
- return lb;
- }
-
- public static Token<BlockTokenIdentifier> convert(
- TokenProto blockToken) {
- return new Token<BlockTokenIdentifier>(blockToken.getIdentifier()
- .toByteArray(), blockToken.getPassword().toByteArray(), new Text(
- blockToken.getKind()), new Text(blockToken.getService()));
- }
-
-
- public static Token<DelegationTokenIdentifier> convertDelegationToken(
- TokenProto blockToken) {
- return new Token<DelegationTokenIdentifier>(blockToken.getIdentifier()
- .toByteArray(), blockToken.getPassword().toByteArray(), new Text(
- blockToken.getKind()), new Text(blockToken.getService()));
- }
-
public static ReplicaState convert(ReplicaStateProto state) {
switch (state) {
case RBW:
@@ -786,15 +386,15 @@ public class PBHelper {
DatanodeRegistrationProto.Builder builder = DatanodeRegistrationProto
.newBuilder();
return builder.setDatanodeID(PBHelperClient.convert((DatanodeID) registration))
- .setStorageInfo(PBHelper.convert(registration.getStorageInfo()))
- .setKeys(PBHelper.convert(registration.getExportedKeys()))
+ .setStorageInfo(convert(registration.getStorageInfo()))
+ .setKeys(convert(registration.getExportedKeys()))
.setSoftwareVersion(registration.getSoftwareVersion()).build();
}
public static DatanodeRegistration convert(DatanodeRegistrationProto proto) {
StorageInfo si = convert(proto.getStorageInfo(), NodeType.DATA_NODE);
- return new DatanodeRegistration(PBHelper.convert(proto.getDatanodeID()),
- si, PBHelper.convert(proto.getKeys()), proto.getSoftwareVersion());
+ return new DatanodeRegistration(PBHelperClient.convert(proto.getDatanodeID()),
+ si, convert(proto.getKeys()), proto.getSoftwareVersion());
}
public static DatanodeCommand convert(DatanodeCommandProto proto) {
@@ -826,7 +426,7 @@ public class PBHelper {
public static KeyUpdateCommandProto convert(KeyUpdateCommand cmd) {
return KeyUpdateCommandProto.newBuilder()
- .setKeys(PBHelper.convert(cmd.getExportedKeys())).build();
+ .setKeys(convert(cmd.getExportedKeys())).build();
}
public static BlockRecoveryCommandProto convert(BlockRecoveryCommand cmd) {
@@ -861,29 +461,17 @@ public class PBHelper {
}
Block[] blocks = cmd.getBlocks();
for (int i = 0; i < blocks.length; i++) {
- builder.addBlocks(PBHelper.convert(blocks[i]));
+ builder.addBlocks(PBHelperClient.convert(blocks[i]));
}
- builder.addAllTargets(convert(cmd.getTargets()))
+ builder.addAllTargets(PBHelperClient.convert(cmd.getTargets()))
.addAllTargetStorageUuids(convert(cmd.getTargetStorageIDs()));
StorageType[][] types = cmd.getTargetStorageTypes();
if (types != null) {
- builder.addAllTargetStorageTypes(convert(types));
+ builder.addAllTargetStorageTypes(PBHelperClient.convert(types));
}
return builder.build();
}
- private static List<StorageTypesProto> convert(StorageType[][] types) {
- List<StorageTypesProto> list = Lists.newArrayList();
- if (types != null) {
- for (StorageType[] ts : types) {
- StorageTypesProto.Builder builder = StorageTypesProto.newBuilder();
- builder.addAllStorageTypes(PBHelperClient.convertStorageTypes(ts));
- list.add(builder.build());
- }
- }
- return list;
- }
-
public static BlockIdCommandProto convert(BlockIdCommand cmd) {
BlockIdCommandProto.Builder builder = BlockIdCommandProto.newBuilder()
.setBlockPoolId(cmd.getBlockPoolId());
@@ -904,15 +492,6 @@ public class PBHelper {
return builder.build();
}
- private static List<DatanodeInfosProto> convert(DatanodeInfo[][] targets) {
- DatanodeInfosProto[] ret = new DatanodeInfosProto[targets.length];
- for (int i = 0; i < targets.length; i++) {
- ret[i] = DatanodeInfosProto.newBuilder()
- .addAllDatanodes(PBHelperClient.convert(targets[i])).build();
- }
- return Arrays.asList(ret);
- }
-
private static List<StorageUuidsProto> convert(String[][] targetStorageUuids) {
StorageUuidsProto[] ret = new StorageUuidsProto[targetStorageUuids.length];
for (int i = 0; i < targetStorageUuids.length; i++) {
@@ -971,7 +550,7 @@ public class PBHelper {
}
public static KeyUpdateCommand convert(KeyUpdateCommandProto keyUpdateCmd) {
- return new KeyUpdateCommand(PBHelper.convert(keyUpdateCmd.getKeys()));
+ return new KeyUpdateCommand(convert(keyUpdateCmd.getKeys()));
}
public static FinalizeCommand convert(FinalizeCommandProto finalizeCmd) {
@@ -994,12 +573,12 @@ public class PBHelper {
List<BlockProto> blockProtoList = blkCmd.getBlocksList();
Block[] blocks = new Block[blockProtoList.size()];
for (int i = 0; i < blockProtoList.size(); i++) {
- blocks[i] = PBHelper.convert(blockProtoList.get(i));
+ blocks[i] = PBHelperClient.convert(blockProtoList.get(i));
}
List<DatanodeInfosProto> targetList = blkCmd.getTargetsList();
DatanodeInfo[][] targets = new DatanodeInfo[targetList.size()][];
for (int i = 0; i < targetList.size(); i++) {
- targets[i] = PBHelper.convert(targetList.get(i));
+ targets[i] = PBHelperClient.convert(targetList.get(i));
}
StorageType[][] targetStorageTypes = new StorageType[targetList.size()][];
@@ -1012,7 +591,7 @@ public class PBHelper {
} else {
for(int i = 0; i < targetStorageTypes.length; i++) {
List<StorageTypeProto> p = targetStorageTypesList.get(i).getStorageTypesList();
- targetStorageTypes[i] = convertStorageTypes(p, targets[i].length);
+ targetStorageTypes[i] = PBHelperClient.convertStorageTypes(p, targets[i].length);
}
}
@@ -1061,15 +640,6 @@ public class PBHelper {
return new BlockIdCommand(action, blkIdCmd.getBlockPoolId(), blockIds);
}
- public static DatanodeInfo[] convert(DatanodeInfosProto datanodeInfosProto) {
- List<DatanodeInfoProto> proto = datanodeInfosProto.getDatanodesList();
- DatanodeInfo[] infos = new DatanodeInfo[proto.size()];
- for (int i = 0; i < infos.length; i++) {
- infos[i] = PBHelper.convert(proto.get(i));
- }
- return infos;
- }
-
public static BalancerBandwidthCommand convert(
BalancerBandwidthCommandProto balancerCmd) {
return new BalancerBandwidthCommand(balancerCmd.getBandwidth());
@@ -1100,8 +670,8 @@ public class PBHelper {
if (receivedDeletedBlockInfo.getDelHints() != null) {
builder.setDeleteHint(receivedDeletedBlockInfo.getDelHints());
}
- return builder.setBlock(PBHelper.convert(receivedDeletedBlockInfo.getBlock()))
- .build();
+ return builder.setBlock(
+ PBHelperClient.convert(receivedDeletedBlockInfo.getBlock())).build();
}
public static ReceivedDeletedBlockInfo convert(
@@ -1119,7 +689,7 @@ public class PBHelper {
break;
}
return new ReceivedDeletedBlockInfo(
- PBHelper.convert(proto.getBlock()),
+ PBHelperClient.convert(proto.getBlock()),
status,
proto.hasDeleteHint() ? proto.getDeleteHint() : null);
}
@@ -1134,539 +704,7 @@ public class PBHelper {
.setCapabilities(info.getCapabilities())
.build();
}
-
- // Located Block Arrays and Lists
- public static LocatedBlockProto[] convertLocatedBlock(LocatedBlock[] lb) {
- if (lb == null) return null;
- return convertLocatedBlock2(Arrays.asList(lb)).toArray(
- new LocatedBlockProto[lb.length]);
- }
-
- public static LocatedBlock[] convertLocatedBlock(LocatedBlockProto[] lb) {
- if (lb == null) return null;
- return convertLocatedBlock(Arrays.asList(lb)).toArray(
- new LocatedBlock[lb.length]);
- }
-
- public static List<LocatedBlock> convertLocatedBlock(
- List<LocatedBlockProto> lb) {
- if (lb == null) return null;
- final int len = lb.size();
- List<LocatedBlock> result =
- new ArrayList<LocatedBlock>(len);
- for (int i = 0; i < len; ++i) {
- result.add(PBHelper.convert(lb.get(i)));
- }
- return result;
- }
-
- public static List<LocatedBlockProto> convertLocatedBlock2(List<LocatedBlock> lb) {
- if (lb == null) return null;
- final int len = lb.size();
- List<LocatedBlockProto> result = new ArrayList<LocatedBlockProto>(len);
- for (int i = 0; i < len; ++i) {
- result.add(PBHelper.convert(lb.get(i)));
- }
- return result;
- }
-
-
- // LocatedBlocks
- public static LocatedBlocks convert(LocatedBlocksProto lb) {
- return new LocatedBlocks(
- lb.getFileLength(), lb.getUnderConstruction(),
- PBHelper.convertLocatedBlock(lb.getBlocksList()),
- lb.hasLastBlock() ? PBHelper.convert(lb.getLastBlock()) : null,
- lb.getIsLastBlockComplete(),
- lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) :
- null);
- }
-
- public static LocatedBlocksProto convert(LocatedBlocks lb) {
- if (lb == null) {
- return null;
- }
- LocatedBlocksProto.Builder builder =
- LocatedBlocksProto.newBuilder();
- if (lb.getLastLocatedBlock() != null) {
- builder.setLastBlock(PBHelper.convert(lb.getLastLocatedBlock()));
- }
- if (lb.getFileEncryptionInfo() != null) {
- builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
- }
- return builder.setFileLength(lb.getFileLength())
- .setUnderConstruction(lb.isUnderConstruction())
- .addAllBlocks(PBHelper.convertLocatedBlock2(lb.getLocatedBlocks()))
- .setIsLastBlockComplete(lb.isLastBlockComplete()).build();
- }
-
- // DataEncryptionKey
- public static DataEncryptionKey convert(DataEncryptionKeyProto bet) {
- String encryptionAlgorithm = bet.getEncryptionAlgorithm();
- return new DataEncryptionKey(bet.getKeyId(),
- bet.getBlockPoolId(),
- bet.getNonce().toByteArray(),
- bet.getEncryptionKey().toByteArray(),
- bet.getExpiryDate(),
- encryptionAlgorithm.isEmpty() ? null : encryptionAlgorithm);
- }
-
- public static DataEncryptionKeyProto convert(DataEncryptionKey bet) {
- DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
- .setKeyId(bet.keyId)
- .setBlockPoolId(bet.blockPoolId)
- .setNonce(ByteString.copyFrom(bet.nonce))
- .setEncryptionKey(ByteString.copyFrom(bet.encryptionKey))
- .setExpiryDate(bet.expiryDate);
- if (bet.encryptionAlgorithm != null) {
- b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
- }
- return b.build();
- }
-
- public static FsServerDefaults convert(FsServerDefaultsProto fs) {
- if (fs == null) return null;
- return new FsServerDefaults(
- fs.getBlockSize(), fs.getBytesPerChecksum(),
- fs.getWritePacketSize(), (short) fs.getReplication(),
- fs.getFileBufferSize(),
- fs.getEncryptDataTransfer(),
- fs.getTrashInterval(),
- PBHelperClient.convert(fs.getChecksumType()));
- }
-
- public static FsServerDefaultsProto convert(FsServerDefaults fs) {
- if (fs == null) return null;
- return FsServerDefaultsProto.newBuilder().
- setBlockSize(fs.getBlockSize()).
- setBytesPerChecksum(fs.getBytesPerChecksum()).
- setWritePacketSize(fs.getWritePacketSize())
- .setReplication(fs.getReplication())
- .setFileBufferSize(fs.getFileBufferSize())
- .setEncryptDataTransfer(fs.getEncryptDataTransfer())
- .setTrashInterval(fs.getTrashInterval())
- .setChecksumType(PBHelperClient.convert(fs.getChecksumType()))
- .build();
- }
-
- public static FsPermissionProto convert(FsPermission p) {
- return FsPermissionProto.newBuilder().setPerm(p.toExtendedShort()).build();
- }
-
- public static FsPermission convert(FsPermissionProto p) {
- return new FsPermissionExtension((short)p.getPerm());
- }
-
-
- // The creatFlag field in PB is a bitmask whose values are the same a the
- // emum values of CreateFlag
- public static int convertCreateFlag(EnumSetWritable<CreateFlag> flag) {
- int value = 0;
- if (flag.contains(CreateFlag.APPEND)) {
- value |= CreateFlagProto.APPEND.getNumber();
- }
- if (flag.contains(CreateFlag.CREATE)) {
- value |= CreateFlagProto.CREATE.getNumber();
- }
- if (flag.contains(CreateFlag.OVERWRITE)) {
- value |= CreateFlagProto.OVERWRITE.getNumber();
- }
- if (flag.contains(CreateFlag.LAZY_PERSIST)) {
- value |= CreateFlagProto.LAZY_PERSIST.getNumber();
- }
- if (flag.contains(CreateFlag.NEW_BLOCK)) {
- value |= CreateFlagProto.NEW_BLOCK.getNumber();
- }
- return value;
- }
-
- public static EnumSetWritable<CreateFlag> convertCreateFlag(int flag) {
- EnumSet<CreateFlag> result =
- EnumSet.noneOf(CreateFlag.class);
- if ((flag & CreateFlagProto.APPEND_VALUE) == CreateFlagProto.APPEND_VALUE) {
- result.add(CreateFlag.APPEND);
- }
- if ((flag & CreateFlagProto.CREATE_VALUE) == CreateFlagProto.CREATE_VALUE) {
- result.add(CreateFlag.CREATE);
- }
- if ((flag & CreateFlagProto.OVERWRITE_VALUE)
- == CreateFlagProto.OVERWRITE_VALUE) {
- result.add(CreateFlag.OVERWRITE);
- }
- if ((flag & CreateFlagProto.LAZY_PERSIST_VALUE)
- == CreateFlagProto.LAZY_PERSIST_VALUE) {
- result.add(CreateFlag.LAZY_PERSIST);
- }
- if ((flag & CreateFlagProto.NEW_BLOCK_VALUE)
- == CreateFlagProto.NEW_BLOCK_VALUE) {
- result.add(CreateFlag.NEW_BLOCK);
- }
- return new EnumSetWritable<CreateFlag>(result, CreateFlag.class);
- }
-
- public static int convertCacheFlags(EnumSet<CacheFlag> flags) {
- int value = 0;
- if (flags.contains(CacheFlag.FORCE)) {
- value |= CacheFlagProto.FORCE.getNumber();
- }
- return value;
- }
-
- public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
- EnumSet<CacheFlag> result = EnumSet.noneOf(CacheFlag.class);
- if ((flags & CacheFlagProto.FORCE_VALUE) == CacheFlagProto.FORCE_VALUE) {
- result.add(CacheFlag.FORCE);
- }
- return result;
- }
-
- public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
- if (fs == null)
- return null;
- return new HdfsLocatedFileStatus(
- fs.getLength(), fs.getFileType().equals(FileType.IS_DIR),
- fs.getBlockReplication(), fs.getBlocksize(),
- fs.getModificationTime(), fs.getAccessTime(),
- PBHelper.convert(fs.getPermission()), fs.getOwner(), fs.getGroup(),
- fs.getFileType().equals(FileType.IS_SYMLINK) ?
- fs.getSymlink().toByteArray() : null,
- fs.getPath().toByteArray(),
- fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID,
- fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null,
- fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
- fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null,
- fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
- : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
- }
-
- public static SnapshottableDirectoryStatus convert(
- SnapshottableDirectoryStatusProto sdirStatusProto) {
- if (sdirStatusProto == null) {
- return null;
- }
- final HdfsFileStatusProto status = sdirStatusProto.getDirStatus();
- return new SnapshottableDirectoryStatus(
- status.getModificationTime(),
- status.getAccessTime(),
- PBHelper.convert(status.getPermission()),
- status.getOwner(),
- status.getGroup(),
- status.getPath().toByteArray(),
- status.getFileId(),
- status.getChildrenNum(),
- sdirStatusProto.getSnapshotNumber(),
- sdirStatusProto.getSnapshotQuota(),
- sdirStatusProto.getParentFullpath().toByteArray());
- }
-
- public static HdfsFileStatusProto convert(HdfsFileStatus fs) {
- if (fs == null)
- return null;
- FileType fType = FileType.IS_FILE;
- if (fs.isDir()) {
- fType = FileType.IS_DIR;
- } else if (fs.isSymlink()) {
- fType = FileType.IS_SYMLINK;
- }
-
- HdfsFileStatusProto.Builder builder =
- HdfsFileStatusProto.newBuilder().
- setLength(fs.getLen()).
- setFileType(fType).
- setBlockReplication(fs.getReplication()).
- setBlocksize(fs.getBlockSize()).
- setModificationTime(fs.getModificationTime()).
- setAccessTime(fs.getAccessTime()).
- setPermission(PBHelper.convert(fs.getPermission())).
- setOwner(fs.getOwner()).
- setGroup(fs.getGroup()).
- setFileId(fs.getFileId()).
- setChildrenNum(fs.getChildrenNum()).
- setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
- setStoragePolicy(fs.getStoragePolicy());
- if (fs.isSymlink()) {
- builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
- }
- if (fs.getFileEncryptionInfo() != null) {
- builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
- }
- if (fs instanceof HdfsLocatedFileStatus) {
- final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
- LocatedBlocks locations = lfs.getBlockLocations();
- if (locations != null) {
- builder.setLocations(PBHelper.convert(locations));
- }
- }
- return builder.build();
- }
-
- public static SnapshottableDirectoryStatusProto convert(
- SnapshottableDirectoryStatus status) {
- if (status == null) {
- return null;
- }
- int snapshotNumber = status.getSnapshotNumber();
- int snapshotQuota = status.getSnapshotQuota();
- byte[] parentFullPath = status.getParentFullPath();
- ByteString parentFullPathBytes = ByteString.copyFrom(
- parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
- HdfsFileStatusProto fs = convert(status.getDirStatus());
- SnapshottableDirectoryStatusProto.Builder builder =
- SnapshottableDirectoryStatusProto
- .newBuilder().setSnapshotNumber(snapshotNumber)
- .setSnapshotQuota(snapshotQuota).setParentFullpath(parentFullPathBytes)
- .setDirStatus(fs);
- return builder.build();
- }
-
- public static HdfsFileStatusProto[] convert(HdfsFileStatus[] fs) {
- if (fs == null) return null;
- final int len = fs.length;
- HdfsFileStatusProto[] result = new HdfsFileStatusProto[len];
- for (int i = 0; i < len; ++i) {
- result[i] = PBHelper.convert(fs[i]);
- }
- return result;
- }
-
- public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) {
- if (fs == null) return null;
- final int len = fs.length;
- HdfsFileStatus[] result = new HdfsFileStatus[len];
- for (int i = 0; i < len; ++i) {
- result[i] = PBHelper.convert(fs[i]);
- }
- return result;
- }
-
- public static DirectoryListing convert(DirectoryListingProto dl) {
- if (dl == null)
- return null;
- List<HdfsFileStatusProto> partList = dl.getPartialListingList();
- return new DirectoryListing(
- partList.isEmpty() ? new HdfsLocatedFileStatus[0]
- : PBHelper.convert(
- partList.toArray(new HdfsFileStatusProto[partList.size()])),
- dl.getRemainingEntries());
- }
-
- public static DirectoryListingProto convert(DirectoryListing d) {
- if (d == null)
- return null;
- return DirectoryListingProto.newBuilder().
- addAllPartialListing(Arrays.asList(
- PBHelper.convert(d.getPartialListing()))).
- setRemainingEntries(d.getRemainingEntries()).
- build();
- }
-
- public static long[] convert(GetFsStatsResponseProto res) {
- long[] result = new long[7];
- result[ClientProtocol.GET_STATS_CAPACITY_IDX] = res.getCapacity();
- result[ClientProtocol.GET_STATS_USED_IDX] = res.getUsed();
- result[ClientProtocol.GET_STATS_REMAINING_IDX] = res.getRemaining();
- result[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] = res.getUnderReplicated();
- result[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = res.getCorruptBlocks();
- result[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = res.getMissingBlocks();
- result[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] =
- res.getMissingReplOneBlocks();
- return result;
- }
-
- public static GetFsStatsResponseProto convert(long[] fsStats) {
- GetFsStatsResponseProto.Builder result = GetFsStatsResponseProto
- .newBuilder();
- if (fsStats.length >= ClientProtocol.GET_STATS_CAPACITY_IDX + 1)
- result.setCapacity(fsStats[ClientProtocol.GET_STATS_CAPACITY_IDX]);
- if (fsStats.length >= ClientProtocol.GET_STATS_USED_IDX + 1)
- result.setUsed(fsStats[ClientProtocol.GET_STATS_USED_IDX]);
- if (fsStats.length >= ClientProtocol.GET_STATS_REMAINING_IDX + 1)
- result.setRemaining(fsStats[ClientProtocol.GET_STATS_REMAINING_IDX]);
- if (fsStats.length >= ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX + 1)
- result.setUnderReplicated(
- fsStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]);
- if (fsStats.length >= ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX + 1)
- result.setCorruptBlocks(
- fsStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX]);
- if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX + 1)
- result.setMissingBlocks(
- fsStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]);
- if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX + 1)
- result.setMissingReplOneBlocks(
- fsStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]);
- return result.build();
- }
-
- public static DatanodeReportTypeProto
- convert(DatanodeReportType t) {
- switch (t) {
- case ALL: return DatanodeReportTypeProto.ALL;
- case LIVE: return DatanodeReportTypeProto.LIVE;
- case DEAD: return DatanodeReportTypeProto.DEAD;
- case DECOMMISSIONING: return DatanodeReportTypeProto.DECOMMISSIONING;
- default:
- throw new IllegalArgumentException("Unexpected data type report:" + t);
- }
- }
-
- public static DatanodeReportType
- convert(DatanodeReportTypeProto t) {
- switch (t) {
- case ALL: return DatanodeReportType.ALL;
- case LIVE: return DatanodeReportType.LIVE;
- case DEAD: return DatanodeReportType.DEAD;
- case DECOMMISSIONING: return DatanodeReportType.DECOMMISSIONING;
- default:
- throw new IllegalArgumentException("Unexpected data type report:" + t);
- }
- }
-
- public static SafeModeActionProto convert(
- SafeModeAction a) {
- switch (a) {
- case SAFEMODE_LEAVE:
- return SafeModeActionProto.SAFEMODE_LEAVE;
- case SAFEMODE_ENTER:
- return SafeModeActionProto.SAFEMODE_ENTER;
- case SAFEMODE_GET:
- return SafeModeActionProto.SAFEMODE_GET;
- default:
- throw new IllegalArgumentException("Unexpected SafeModeAction :" + a);
- }
- }
-
- public static SafeModeAction convert(
- ClientNamenodeProtocolProtos.SafeModeActionProto a) {
- switch (a) {
- case SAFEMODE_LEAVE:
- return SafeModeAction.SAFEMODE_LEAVE;
- case SAFEMODE_ENTER:
- return SafeModeAction.SAFEMODE_ENTER;
- case SAFEMODE_GET:
- return SafeModeAction.SAFEMODE_GET;
- default:
- throw new IllegalArgumentException("Unexpected SafeModeAction :" + a);
- }
- }
-
- public static RollingUpgradeActionProto convert(RollingUpgradeAction a) {
- switch (a) {
- case QUERY:
- return RollingUpgradeActionProto.QUERY;
- case PREPARE:
- return RollingUpgradeActionProto.START;
- case FINALIZE:
- return RollingUpgradeActionProto.FINALIZE;
- default:
- throw new IllegalArgumentException("Unexpected value: " + a);
- }
- }
-
- public static RollingUpgradeAction convert(RollingUpgradeActionProto a) {
- switch (a) {
- case QUERY:
- return RollingUpgradeAction.QUERY;
- case START:
- return RollingUpgradeAction.PREPARE;
- case FINALIZE:
- return RollingUpgradeAction.FINALIZE;
- default:
- throw new IllegalArgumentException("Unexpected value: " + a);
- }
- }
-
- public static RollingUpgradeStatusProto convertRollingUpgradeStatus(
- RollingUpgradeStatus status) {
- return RollingUpgradeStatusProto.newBuilder()
- .setBlockPoolId(status.getBlockPoolId())
- .setFinalized(status.isFinalized())
- .build();
- }
-
- public static RollingUpgradeStatus convert(RollingUpgradeStatusProto proto) {
- return new RollingUpgradeStatus(proto.getBlockPoolId(),
- proto.getFinalized());
- }
-
- public static RollingUpgradeInfoProto convert(RollingUpgradeInfo info) {
- return RollingUpgradeInfoProto.newBuilder()
- .setStatus(convertRollingUpgradeStatus(info))
- .setCreatedRollbackImages(info.createdRollbackImages())
- .setStartTime(info.getStartTime())
- .setFinalizeTime(info.getFinalizeTime())
- .build();
- }
-
- public static RollingUpgradeInfo convert(RollingUpgradeInfoProto proto) {
- RollingUpgradeStatusProto status = proto.getStatus();
- return new RollingUpgradeInfo(status.getBlockPoolId(),
- proto.getCreatedRollbackImages(),
- proto.getStartTime(), proto.getFinalizeTime());
- }
-
- public static CorruptFileBlocks convert(CorruptFileBlocksProto c) {
- if (c == null)
- return null;
- List<String> fileList = c.getFilesList();
- return new CorruptFileBlocks(fileList.toArray(new String[fileList.size()]),
- c.getCookie());
- }
-
- public static CorruptFileBlocksProto convert(CorruptFileBlocks c) {
- if (c == null)
- return null;
- return CorruptFileBlocksProto.newBuilder().
- addAllFiles(Arrays.asList(c.getFiles())).
- setCookie(c.getCookie()).
- build();
- }
-
- public static ContentSummary convert(ContentSummaryProto cs) {
- if (cs == null) return null;
- ContentSummary.Builder builder = new ContentSummary.Builder();
- builder.length(cs.getLength()).
- fileCount(cs.getFileCount()).
- directoryCount(cs.getDirectoryCount()).
- quota(cs.getQuota()).
- spaceConsumed(cs.getSpaceConsumed()).
- spaceQuota(cs.getSpaceQuota());
- if (cs.hasTypeQuotaInfos()) {
- for (HdfsProtos.StorageTypeQuotaInfoProto info :
- cs.getTypeQuotaInfos().getTypeQuotaInfoList()) {
- StorageType type = PBHelperClient.convertStorageType(info.getType());
- builder.typeConsumed(type, info.getConsumed());
- builder.typeQuota(type, info.getQuota());
- }
- }
- return builder.build();
- }
-
- public static ContentSummaryProto convert(ContentSummary cs) {
- if (cs == null) return null;
- ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
- builder.setLength(cs.getLength()).
- setFileCount(cs.getFileCount()).
- setDirectoryCount(cs.getDirectoryCount()).
- setQuota(cs.getQuota()).
- setSpaceConsumed(cs.getSpaceConsumed()).
- setSpaceQuota(cs.getSpaceQuota());
- if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
- HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
- HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
- for (StorageType t: StorageType.getTypesSupportingQuota()) {
- HdfsProtos.StorageTypeQuotaInfoProto info =
- HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
- setType(PBHelperClient.convertStorageType(t)).
- setConsumed(cs.getTypeConsumed(t)).
- setQuota(cs.getTypeQuota(t)).
- build();
- isb.addTypeQuotaInfo(info);
- }
- builder.setTypeQuotaInfos(isb);
- }
- return builder.build();
- }
public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) {
if (s == null) return null;
@@ -1699,95 +737,12 @@ public class PBHelper {
return builder.build();
}
- public static DatanodeStorageProto convert(DatanodeStorage s) {
- return DatanodeStorageProto.newBuilder()
- .setState(PBHelper.convertState(s.getState()))
- .setStorageType(PBHelperClient.convertStorageType(s.getStorageType()))
- .setStorageUuid(s.getStorageID()).build();
- }
-
- private static StorageState convertState(State state) {
- switch(state) {
- case READ_ONLY_SHARED:
- return StorageState.READ_ONLY_SHARED;
- case NORMAL:
- default:
- return StorageState.NORMAL;
- }
- }
-
- public static DatanodeStorage convert(DatanodeStorageProto s) {
- return new DatanodeStorage(s.getStorageUuid(),
- PBHelper.convertState(s.getState()),
- PBHelperClient.convertStorageType(s.getStorageType()));
- }
-
- private static State convertState(StorageState state) {
- switch(state) {
- case READ_ONLY_SHARED:
- return DatanodeStorage.State.READ_ONLY_SHARED;
- case NORMAL:
- default:
- return DatanodeStorage.State.NORMAL;
- }
- }
-
- public static StorageType[] convertStorageTypes(
- List<StorageTypeProto> storageTypesList, int expectedSize) {
- final StorageType[] storageTypes = new StorageType[expectedSize];
- if (storageTypesList.size() != expectedSize) { // missing storage types
- Preconditions.checkState(storageTypesList.isEmpty());
- Arrays.fill(storageTypes, StorageType.DEFAULT);
- } else {
- for (int i = 0; i < storageTypes.length; ++i) {
- storageTypes[i] = PBHelperClient.convertStorageType(storageTypesList.get(i));
- }
- }
- return storageTypes;
- }
-
- public static StorageReportProto convert(StorageReport r) {
- StorageReportProto.Builder builder = StorageReportProto.newBuilder()
- .setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
- .setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
- .setStorageUuid(r.getStorage().getStorageID())
- .setStorage(convert(r.getStorage()));
- return builder.build();
- }
-
- public static StorageReport convert(StorageReportProto p) {
- return new StorageReport(
- p.hasStorage() ?
- convert(p.getStorage()) :
- new DatanodeStorage(p.getStorageUuid()),
- p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
- p.getBlockPoolUsed());
- }
-
- public static StorageReport[] convertStorageReports(
- List<StorageReportProto> list) {
- final StorageReport[] report = new StorageReport[list.size()];
- for (int i = 0; i < report.length; i++) {
- report[i] = convert(list.get(i));
- }
- return report;
- }
-
- public static List<StorageReportProto> convertStorageReports(StorageReport[] storages) {
- final List<StorageReportProto> protos = new ArrayList<StorageReportProto>(
- storages.length);
- for(int i = 0; i < storages.length; i++) {
- protos.add(convert(storages[i]));
- }
- return protos;
- }
-
- public static VolumeFailureSummary convertVolumeFailureSummary(
- VolumeFailureSummaryProto proto) {
- List<String> failedStorageLocations = proto.getFailedStorageLocationsList();
- return new VolumeFailureSummary(
- failedStorageLocations.toArray(new String[failedStorageLocations.size()]),
- proto.getLastVolumeFailureDate(), proto.getEstimatedCapacityLostTotal());
+ public static VolumeFailureSummary convertVolumeFailureSummary(
+ VolumeFailureSummaryProto proto) {
+ List<String> failedStorageLocations = proto.getFailedStorageLocationsList();
+ return new VolumeFailureSummary(
+ failedStorageLocations.toArray(new String[failedStorageLocations.size()]),
+ proto.getLastVolumeFailureDate(), proto.getEstimatedCapacityLostTotal());
}
public static VolumeFailureSummaryProto convertVolumeFailureSummary(
@@ -1819,933 +774,8 @@ public class PBHelper {
return JournalInfoProto.newBuilder().setClusterID(j.getClusterId())
.setLayoutVersion(j.getLayoutVersion())
.setNamespaceID(j.getNamespaceId()).build();
- }
-
- public static SnapshottableDirectoryStatus[] convert(
- SnapshottableDirectoryListingProto sdlp) {
- if (sdlp == null)
- return null;
- List<SnapshottableDirectoryStatusProto> list = sdlp
- .getSnapshottableDirListingList();
- if (list.isEmpty()) {
- return new SnapshottableDirectoryStatus[0];
- } else {
- SnapshottableDirectoryStatus[] result =
- new SnapshottableDirectoryStatus[list.size()];
- for (int i = 0; i < list.size(); i++) {
- result[i] = PBHelper.convert(list.get(i));
- }
- return result;
- }
- }
-
- public static SnapshottableDirectoryListingProto convert(
- SnapshottableDirectoryStatus[] status) {
- if (status == null)
- return null;
- SnapshottableDirectoryStatusProto[] protos =
- new SnapshottableDirectoryStatusProto[status.length];
- for (int i = 0; i < status.length; i++) {
- protos[i] = PBHelper.convert(status[i]);
- }
- List<SnapshottableDirectoryStatusProto> protoList = Arrays.asList(protos);
- return SnapshottableDirectoryListingProto.newBuilder()
- .addAllSnapshottableDirListing(protoList).build();
- }
-
- public static DiffReportEntry convert(SnapshotDiffReportEntryProto entry) {
- if (entry == null) {
- return null;
- }
- DiffType type = DiffType.getTypeFromLabel(entry
- .getModificationLabel());
- return type == null ? null : new DiffReportEntry(type, entry.getFullpath()
- .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath()
- .toByteArray() : null);
- }
-
- public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) {
- if (entry == null) {
- return null;
- }
- ByteString sourcePath = ByteString
- .copyFrom(entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES : entry
- .getSourcePath());
- String modification = entry.getType().getLabel();
- SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
- .newBuilder().setFullpath(sourcePath)
- .setModificationLabel(modification);
- if (entry.getType() == DiffType.RENAME) {
- ByteString targetPath = ByteString
- .copyFrom(entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES : entry
- .getTargetPath());
- builder.setTargetPath(targetPath);
- }
- return builder.build();
- }
-
- public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) {
- if (reportProto == null) {
- return null;
- }
- String snapshotDir = reportProto.getSnapshotRoot();
- String fromSnapshot = reportProto.getFromSnapshot();
- String toSnapshot = reportProto.getToSnapshot();
- List<SnapshotDiffReportEntryProto> list = reportProto
- .getDiffReportEntriesList();
- List<DiffReportEntry> entries = new ArrayList<DiffReportEntry>();
- for (SnapshotDiffReportEntryProto entryProto : list) {
- DiffReportEntry entry = convert(entryProto);
- if (entry != null)
- entries.add(entry);
- }
- return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot,
- entries);
- }
-
- public static SnapshotDiffReportProto convert(SnapshotDiffReport report) {
- if (report == null) {
- return null;
- }
- List<DiffReportEntry> entries = report.getDiffList();
- List<SnapshotDiffReportEntryProto> entryProtos =
- new ArrayList<SnapshotDiffReportEntryProto>();
- for (DiffReportEntry entry : entries) {
- SnapshotDiffReportEntryProto entryProto = convert(entry);
- if (entryProto != null)
- entryProtos.add(entryProto);
- }
-
- SnapshotDiffReportProto reportProto = SnapshotDiffReportProto.newBuilder()
- .setSnapshotRoot(report.getSnapshotRoot())
- .setFromSnapshot(report.getFromSnapshot())
- .setToSnapshot(report.getLaterSnapshotName())
- .addAllDiffReportEntries(entryProtos).build();
- return reportProto;
- }
-
- public static CacheDirectiveInfoProto convert
- (CacheDirectiveInfo info) {
- CacheDirectiveInfoProto.Builder builder =
- CacheDirectiveInfoProto.newBuilder();
- if (info.getId() != null) {
- builder.setId(info.getId());
- }
- if (info.getPath() != null) {
- builder.setPath(info.getPath().toUri().getPath());
- }
- if (info.getReplication() != null) {
- builder.setReplication(info.getReplication());
- }
- if (info.getPool() != null) {
- builder.setPool(info.getPool());
- }
- if (info.getExpiration() != null) {
- builder.setExpiration(convert(info.getExpiration()));
- }
- return builder.build();
- }
-
- public static CacheDirectiveInfo convert
- (CacheDirectiveInfoProto proto) {
- CacheDirectiveInfo.Builder builder =
- new CacheDirectiveInfo.Builder();
- if (proto.hasId()) {
- builder.setId(proto.getId());
- }
- if (proto.hasPath()) {
- builder.setPath(new Path(proto.getPath()));
- }
- if (proto.hasReplication()) {
- builder.setReplication(Shorts.checkedCast(
- proto.getReplication()));
- }
- if (proto.hasPool()) {
- builder.setPool(proto.getPool());
- }
- if (proto.hasExpiration()) {
- builder.setExpiration(convert(proto.getExpiration()));
- }
- return builder.build();
- }
-
- public static CacheDirectiveInfoExpirationProto convert(
- CacheDirectiveInfo.Expiration expiration) {
- return CacheDirectiveInfoExpirationProto.newBuilder()
- .setIsRelative(expiration.isRelative())
- .setMillis(expiration.getMillis())
- .build();
- }
-
- public static CacheDirectiveInfo.Expiration convert(
- CacheDirectiveInfoExpirationProto proto) {
- if (proto.getIsRelative()) {
- return CacheDirectiveInfo.Expiration.newRelative(proto.getMillis());
- }
- return CacheDirectiveInfo.Expiration.newAbsolute(proto.getMillis());
- }
-
- public static CacheDirectiveStatsProto convert(CacheDirectiveStats stats) {
- CacheDirectiveStatsProto.Builder builder =
- CacheDirectiveStatsProto.newBuilder();
- builder.setBytesNeeded(stats.getBytesNeeded());
- builder.setBytesCached(stats.getBytesCached());
- builder.setFilesNeeded(stats.getFilesNeeded());
- builder.setFilesCached(stats.getFilesCached());
- builder.setHasExpired(stats.hasExpired());
- return builder.build();
- }
-
- public static CacheDirectiveStats convert(CacheDirectiveStatsProto proto) {
- CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder();
- builder.setBytesNeeded(proto.getBytesNeeded());
- builder.setBytesCached(proto.getBytesCached());
- builder.setFilesNeeded(proto.getFilesNeeded());
- builder.setFilesCached(proto.getFilesCached());
- builder.setHasExpired(proto.getHasExpired());
- return builder.build();
- }
-
- public static CacheDirectiveEntryProto convert(CacheDirectiveEntry entry) {
- CacheDirectiveEntryProto.Builder builder =
- CacheDirectiveEntryProto.newBuilder();
- builder.setInfo(PBHelper.convert(entry.getInfo()));
- builder.setStats(PBHelper.convert(entry.getStats()));
- return builder.build();
- }
-
- public static CacheDirectiveEntry convert(CacheDirectiveEntryProto proto) {
- CacheDirectiveInfo info = PBHelper.convert(proto.getInfo());
- CacheDirectiveStats stats = PBHelper.convert(proto.getStats());
- return new CacheDirectiveEntry(info, stats);
- }
-
- public static CachePoolInfoProto convert(CachePoolInfo info) {
- CachePoolInfoProto.Builder builder = CachePoolInfoProto.newBuilder();
- builder.setPoolName(info.getPoolName());
- if (info.getOwnerName() != null) {
- builder.setOwnerName(info.getOwnerName());
- }
- if (info.getGroupName() != null) {
- builder.setGroupName(info.getGroupName());
- }
- if (info.getMode() != null) {
- builder.setMode(info.getMode().toShort());
- }
- if (info.getLimit() != null) {
- builder.setLimit(info.getLimit());
- }
- if (info.getMaxRelativeExpiryMs() != null) {
- builder.setMaxRelativeExpiry(info.getMaxRelativeExpiryMs());
- }
- return builder.build();
- }
-
- public static CachePoolInfo convert (CachePoolInfoProto proto) {
- // Pool name is a required field, the rest are optional
- String poolName = checkNotNull(proto.getPoolName());
- CachePoolInfo info = new CachePoolInfo(poolName);
- if (proto.hasOwnerName()) {
- info.setOwnerName(proto.getOwnerName());
- }
- if (proto.hasGroupName()) {
- info.setGroupName(proto.getGroupName());
- }
- if (proto.hasMode()) {
- info.setMode(new FsPermission((short)proto.getMode()));
- }
- if (proto.hasLimit()) {
- info.setLimit(proto.getLimit());
- }
- if (proto.hasMaxRelativeExpiry()) {
- info.setMaxRelativeExpiryMs(proto.getMaxRelativeExpiry());
- }
- return info;
- }
-
- public static CachePoolStatsProto convert(CachePoolStats stats) {
- CachePoolStatsProto.Builder builder = CachePoolStatsProto.newBuilder();
- builder.setBytesNeeded(stats.getBytesNeeded());
- builder.setBytesCached(stats.getBytesCached());
- builder.setBytesOverlimit(stats.getBytesOverlimit());
- builder.setFilesNeeded(stats.getFilesNeeded());
- builder.setFilesCached(stats.getFilesCached());
- return builder.build();
- }
-
- public static CachePoolStats convert (CachePoolStatsProto proto) {
- CachePoolStats.Builder builder = new CachePoolStats.Builder();
- builder.setBytesNeeded(proto.getBytesNeeded());
- builder.setBytesCached(proto.getBytesCached());
- builder.setBytesOverlimit(proto.getBytesOverlimit());
- builder.setFilesNeeded(proto.getFilesNeeded());
- builder.setFilesCached(proto.getFilesCached());
- return builder.build();
- }
-
- public static CachePoolEntryProto convert(CachePoolEntry entry) {
- CachePoolEntryProto.Builder builder = CachePoolEntryProto.newBuilder();
- builder.setInfo(PBHelper.convert(entry.getInfo()));
- builder.setStats(PBHelper.convert(entry.getStats()));
- return builder.build();
- }
-
- public static CachePoolEntry convert (CachePoolEntryProto proto) {
- CachePoolInfo info = PBHelper.convert(proto.getInfo());
- CachePoolStats stats = PBHelper.convert(proto.getStats());
- return new CachePoolEntry(info, stats);
- }
-
-
- public static DatanodeLocalInfoProto convert(DatanodeLocalInfo info) {
- DatanodeLocalInfoProto.Builder builder = DatanodeLocalInfoProto.newBuilder();
- builder.setSoftwareVersion(info.getSoftwareVersion());
- builder.setConfigVersion(info.getConfigVersion());
- builder.setUptime(info.getUptime());
- return builder.build();
- }
-
- private static AclEntryScopeProto convert(AclEntryScope v) {
- return AclEntryScopeProto.valueOf(v.ordinal());
- }
-
- private static AclEntryScope convert(AclEntryScopeProto v) {
- return castEnum(v, ACL_ENTRY_SCOPE_VALUES);
- }
-
- private static AclEntryTypeProto convert(AclEntryType e) {
- return AclEntryTypeProto.valueOf(e.ordinal());
- }
-
- private static AclEntryType convert(AclEntryTypeProto v) {
- return castEnum(v, ACL_ENTRY_TYPE_VALUES);
- }
-
- private static XAttrNamespaceProto convert(XAttr.NameSpace v) {
- return XAttrNamespaceProto.valueOf(v.ordinal());
- }
-
- private static XAttr.NameSpace convert(XAttrNamespaceProto v) {
- return castEnum(v, XATTR_NAMESPACE_VALUES);
- }
-
- public static FsActionProto convert(FsAction v) {
- return FsActionProto.valueOf(v != null ? v.ordinal() : 0);
- }
-
- public static FsAction convert(FsActionProto v) {
- return castEnum(v, FSACTION_VALUES);
}
- public static List<AclEntryProto> convertAclEntryProto(
- List<AclEntry> aclSpec) {
- ArrayList<AclEntryProto> r = Lists.newArrayListWithCapacity(aclSpec.size());
- for (AclEntry e : aclSpec) {
- AclEntryProto.Builder builder = AclEntryProto.newBuilder();
- builder.setType(convert(e.getType()));
- builder.setScope(convert(e.getScope()));
- builder.setPermissions(convert(e.getPermission()));
- if (e.getName() != null) {
- builder.setName(e.getName());
- }
- r.add(builder.build());
- }
- return r;
- }
-
- public static List<AclEntry> convertAclEntry(List<AclEntryProto> aclSpec) {
- ArrayList<AclEntry> r = Lists.newArrayListWithCapacity(aclSpec.size());
- for (AclEntryProto e : aclSpec) {
- AclEntry.Builder builder = new AclEntry.Builder();
- builder.setType(convert(e.getType()));
- builder.setScope(convert(e.getScope()));
- builder.setPermission(convert(e.getPermissions()));
- if (e.hasName()) {
- builder.setName(e.getName());
- }
- r.add(builder.build());
- }
- return r;
- }
-
- public static AclStatus convert(GetAclStatusResponseProto e) {
- AclStatusProto r = e.getResult();
- AclStatus.Builder builder = new AclStatus.Builder();
- builder.owner(r.getOwner()).group(r.getGroup()).stickyBit(r.getSticky())
- .addEntries(convertAclEntry(r.getEntriesList()));
- if (r.hasPermission()) {
- builder.setPermission(convert(r.getPermission()));
- }
- return builder.build();
- }
-
- public static GetAclStatusResponseProto convert(AclStatus e) {
- AclStatusProto.Builder builder = AclStatusProto.newBuilder();
- builder.setOwner(e.getOwner())
- .setGroup(e.getGroup()).setSticky(e.isStickyBit())
- .addAllEntries(convertAclEntryProto(e.getEntries()));
- if (e.getPermission() != null) {
- builder.setPermission(convert(e.getPermission()));
- }
- AclStatusProto r = builder.build();
- return GetAclStatusResponseProto.newBuilder().setResult(r).build();
- }
-
- public static XAttrProto convertXAttrProto(XAttr a) {
- XAttrProto.Builder builder = XAttrProto.newBuilder();
- builder.setNamespace(convert(a.getNameSpace()));
- if (a.getName() != null) {
- builder.setName(a.getName());
- }
- if (a.getValue() != null) {
- builder.setValue(getByteString(a.getValue()));
- }
- return builder.build();
- }
-
- public static List<XAttrProto> convertXAttrProto(
- List<XAttr> xAttrSpec) {
- if (xAttrSpec == null) {
- return Lists.newArrayListWithCapacity(0);
- }
- ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
- xAttrSpec.size());
- for (XAttr a : xAttrSpec) {
- XAttrProto.Builder builder = XAttrProto.newBuilder();
- builder.setNamespace(convert(a.getNameSpace()));
- if (a.getName() != null) {
- builder.setName(a.getName());
- }
- if (a.getValue() != null) {
- builder.setValue(getByteString(a.getValue()));
- }
- xAttrs.add(builder.build());
- }
- return xAttrs;
- }
-
- /**
- * The flag field in PB is a bitmask whose values are the same a the
- * emum values of XAttrSetFlag
- */
- public static int convert(EnumSet<XAttrSetFlag> flag) {
- int value = 0;
- if (flag.contains(XAttrSetFlag.CREATE)) {
- value |= XAttrSetFlagProto.XATTR_CREATE.getNumber();
- }
- if (flag.contains(XAttrSetFlag.REPLACE)) {
- value |= XAttrSetFlagProto.XATTR_REPLACE.getNumber();
- }
- return value;
- }
-
- public static EnumSet<XAttrSetFlag> convert(int flag) {
- EnumSet<XAttrSetFlag> result =
- EnumSet.noneOf(XAttrSetFlag.class);
- if ((flag & XAttrSetFlagProto.XATTR_CREATE_VALUE) ==
- XAttrSetFlagProto.XATTR_CREATE_VALUE) {
- result.add(XAttrSetFlag.CREATE);
- }
- if ((flag & XAttrSetFlagProto.XATTR_REPLACE_VALUE) ==
- XAttrSetFlagProto.XATTR_REPLACE_VALUE) {
- result.add(XAttrSetFlag.REPLACE);
- }
- return result;
- }
-
- public static XAttr convertXAttr(XAttrProto a) {
- XAttr.Builder builder = new XAttr.Builder();
- builder.setNameSpace(convert(a.getNamespace()));
- if (a.hasName()) {
- builder.setName(a.getName());
- }
- if (a.hasValue()) {
- builder.setValue(a.getValue().toByteArray());
- }
- return builder.build();
- }
-
- public static List<XAttr> convertXAttrs(List<XAttrProto> xAttrSpec) {
- ArrayList<XAttr> xAttrs = Lists.newArrayListWithCapacity(xAttrSpec.size());
- for (XAttrProto a : xAttrSpec) {
- XAttr.Builder builder = new XAttr.Builder();
- builder.setNameSpace(convert(a.getNamespace()));
- if (a.hasName()) {
- builder.setName(a.getName());
- }
- if (a.hasValue()) {
- builder.setValue(a.getValue().toByteArray());
- }
- xAttrs.add(builder.build());
- }
- return xAttrs;
- }
-
- public static List<XAttr> convert(GetXAttrsResponseProto a) {
- List<XAttrProto> xAttrs = a.getXAttrsList();
- return convertXAttrs(xAttrs);
- }
-
- public static GetXAttrsResponseProto convertXAttrsResponse(
- List<XAttr> xAttrs) {
- GetXAttrsResponseProto.Builder builder = GetXAttrsResponseProto
- .newBuilder();
- if (xAttrs != null) {
- builder.addAllXAttrs(convertXAttrProto(xAttrs));
- }
- return builder.build();
- }
-
- public static List<XAttr> convert(ListXAttrsResponseProto a) {
- final List<XAttrProto> xAttrs = a.getXAttrsList();
- return convertXAttrs(xAttrs);
- }
-
- public static ListXAttrsResponseProto convertListXAttrsResponse(
- List<XAttr> names) {
- ListXAttrsResponseProto.Builder builder =
- ListXAttrsResponseProto.newBuilder();
- if (names != null) {
- builder.addAllXAttrs(convertXAttrProto(names));
- }
- return builder.build();
- }
-
- public static EncryptionZoneProto convert(EncryptionZone zone) {
- return EncryptionZoneProto.newBuilder()
- .setId(zone.getId())
- .setPath(zone.getPath())
- .setSuite(PBHelperClient.convert(zone.getSuite()))
- .setCryptoProtocolVersion(convert(zone.getVersion()))
- .setKeyName(zone.getKeyName())
- .build();
- }
-
- public static EncryptionZone convert(EncryptionZoneProto proto) {
- return new EncryptionZone(proto.getId(), proto.getPath(),
- PBHelperClient.convert(proto.getSuite()), convert(proto.getCryptoProtocolVersion()),
- proto.getKeyName());
- }
-
- public static SlotId convert(ShortCircuitShmSlotProto slotId) {
- return new SlotId(PBHelperClient.convert(slotId.getShmId()),
- slotId.getSlotIdx());
- }
-
- private static Event.CreateEvent.INodeType createTypeConvert(InotifyProtos.INodeType
- type) {
- switch (type) {
- case I_TYPE_DIRECTORY:
- return Event.CreateEvent.INodeType.DIRECTORY;
- case I_TYPE_FILE:
- return Event.CreateEvent.INodeType.FILE;
- case I_TYPE_SYMLINK:
- return Event.CreateEvent.INodeType.SYMLINK;
- default:
- return null;
- }
- }
-
- private static InotifyProtos.MetadataUpdateType metadataUpdateTypeConvert(
- Event.MetadataUpdateEvent.MetadataType type) {
- switch (type) {
- case TIMES:
- return InotifyProtos.MetadataUpdateType.META_TYPE_TIMES;
- case REPLICATION:
- return InotifyProtos.MetadataUpdateType.META_TYPE_REPLICATION;
- case OWNER:
- return InotifyProtos.MetadataUpdateType.META_TYPE_OWNER;
- case PERMS:
- return InotifyProtos.MetadataUpdateType.META_TYPE_PERMS;
- case ACLS:
- return InotifyProtos.MetadataUpdateType.META_TYPE_ACLS;
- case XATTRS:
- return InotifyProtos.MetadataUpdateType.META_TYPE_XATTRS;
- default:
- return null;
- }
- }
-
- private static Event.MetadataUpdateEvent.MetadataType metadataUpdateTypeConvert(
- InotifyProtos.MetadataUpdateType type) {
- switch (type) {
- case META_TYPE_TIMES:
- return Event.MetadataUpdateEvent.MetadataType.TIMES;
- case META_TYPE_REPLICATION:
- return Event.MetadataUpdateEvent.MetadataType.REPLICATION;
- case META_TYPE_OWNER:
- return Event.MetadataUpdateEvent.MetadataType.OWNER;
- case META_TYPE_PERMS:
- return Event.MetadataUpdateEvent.MetadataType.PERMS;
- case META_TYPE_ACLS:
- return Event.MetadataUpdateEvent.MetadataType.ACLS;
- case META_TYPE_XATTRS:
- return Event.MetadataUpdateEvent.MetadataType.XATTRS;
- default:
- return null;
- }
- }
-
- private static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INodeType
- type) {
- switch (type) {
- case DIRECTORY:
- return InotifyProtos.INodeType.I_TYPE_DIRECTORY;
- case FILE:
- return InotifyProtos.INodeType.I_TYPE_FILE;
- case SYMLINK:
- return InotifyProtos.INodeType.I_TYPE_SYMLINK;
- default:
- return null;
- }
- }
-
- public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
- IOException {
- final InotifyProtos.EventsListProto list = resp.getEventsList();
- final long firstTxid = list.getFirstTxid();
- final long lastTxid = list.getLastTxid();
-
- List<EventBatch> batches = Lists.newArrayList();
- if (list.getEventsList().size() > 0) {
- throw new IOException("Can't handle old inotify server response.");
- }
- for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
- long txid = bp.getTxid();
- if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
- throw new IOException("Error converting TxidResponseProto: got a " +
- "transaction id " + txid + " that was outside the range of [" +
- firstTxid + ", " + lastTxid + "].");
- }
- List<Event> events = Lists.newArrayList();
- for (InotifyProtos.EventProto p : bp.getEventsList()) {
- switch (p.getType()) {
- case EVENT_CLOSE:
- InotifyProtos.CloseEventProto close =
- InotifyProtos.CloseEventProto.parseFrom(p.getContents());
- events.add(new Event.CloseEvent(close.getPath(),
- close.getFileSize(), close.getTimestamp()));
- break;
- case EVENT_CREATE:
- InotifyProtos.CreateEventProto create =
- InotifyProtos.CreateEventProto.parseFrom(p.getContents());
- events.add(new Event.CreateEvent.Builder()
- .iNodeType(createTypeConvert(create.getType()))
- .path(create.getPath())
- .ctime(create.getCtime())
- .ownerName(create.getOwnerName())
- .groupName(create.getGroupName())
- .perms(convert(create.getPerms()))
- .replication(create.getReplication())
- .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
- create.getSymlinkTarget())
- .defaultBlockSize(create.getDefaultBlockSize())
- .overwrite(create.getOverwrite()).build());
- break;
- case EVENT_METADATA:
- InotifyProtos.MetadataUpdateEventProto meta =
- InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
- events.add(new Event.MetadataUpdateEvent.Builder()
- .path(meta.getPath())
- .metadataType(metadataUpdateTypeConvert(meta.getType()))
- .mtime(meta.getMtime())
- .atime(meta.getAtime())
- .replication(meta.getReplication())
- .ownerName(
- meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
- .groupName(
- meta.getGroupName().isEmpty() ? null : meta.getGroupName())
- .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
- .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
- meta.getAclsList()))
- .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
- meta.getXAttrsList()))
- .xAttrsRemoved(meta.getXAttrsRemoved())
- .build());
- break;
- case EVENT_RENAME:
- InotifyProtos.RenameEventProto rename =
- InotifyProtos.RenameEventProto.parseFrom(p.getContents());
- events.add(new Event.RenameEvent.Builder()
- .srcPath(rename.getSrcPath())
- .dstPath(rename.getDestPath())
- .timestamp(rename.getTimestamp())
- .build());
- break;
- case EVENT_APPEND:
- InotifyProtos.AppendEventProto append =
- InotifyProtos.AppendEventProto.parseFrom(p.getContents());
- events.add(new Event.AppendEvent.Builder().path(append.getPath())
- .newBlock(append.hasNewBlock() && append.getNewBlock())
- .build());
- break;
- case EVENT_UNLINK:
- InotifyProtos.UnlinkEventProto unlink =
- InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
- events.add(new Event.UnlinkEvent.Builder()
- .path(unlink.getPath())
- .timestamp(unlink.getTimestamp())
- .build());
- break;
- case EVENT_TRUNCATE:
- InotifyProtos.TruncateEventProto truncate =
- InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
- events.add(new Event.TruncateEvent(truncate.getPath(),
- truncate.getFileSize(), truncate.getTimestamp()));
- break;
- default:
- throw new RuntimeException("Unexpected inotify event type: " +
- p.getType());
- }
- }
- batches.add(new EventBatch(txid, events.toArray(new Event[0])));
- }
- return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
- resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
- }
-
- public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList el) {
- InotifyProtos.EventsListProto.Builder builder =
- InotifyProtos.EventsListProto.newBuilder();
- for (EventBatch b : el.getBatches()) {
- List<InotifyProtos.EventProto> events = Lists.newArrayList();
- for (Event e : b.getEvents()) {
- switch (e.getEventType()) {
- case CLOSE:
- Event.CloseEvent ce = (Event.CloseEvent) e;
- events.add(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_CLOSE)
- .setContents(
- InotifyProtos.CloseEventProto.newBuilder()
- .setPath(ce.getPath())
- .setFileSize(ce.getFileSize())
- .setTimestamp(ce.getTimestamp()).build().toByteString()
- ).build());
- break;
- case CREATE:
- Event.CreateEvent ce2 = (Event.CreateEvent) e;
- events.add(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_CREATE)
- .setContents(
- InotifyProtos.CreateEventProto.newBuilder()
- .setType(createTypeConvert(ce2.getiNodeType()))
- .setPath(ce2.getPath())
- .setCtime(ce2.getCtime())
- .setOwnerName(ce2.getOwnerName())
- .setGroupName(ce2.getGroupName())
- .setPerms(convert(ce2.getPerms()))
- .setReplication(ce2.getReplication())
- .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
- "" : ce2.getSymlinkTarget())
- .setDefaultBlockSize(ce2.getDefaultBlockSize())
- .setOverwrite(ce2.getOverwrite()).build().toByteString()
- ).build());
- break;
- case METADATA:
- Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
- InotifyProtos.MetadataUpdateEventProto.Builder metaB =
- InotifyProtos.MetadataUpdateEventProto.newBuilder()
- .setPath(me.getPath())
- .setType(metadataUpdateTypeConvert(me.getMetadataType()))
- .setMtime(me.getMtime())
- .setAtime(me.getAtime())
- .setReplication(me.getReplication())
- .setOwnerName(me.getOwnerName() == null ? "" :
- me.getOwnerName())
- .setGroupName(me.getGroupName() == null ? "" :
- me.getGroupName())
- .addAllAcls(me.getAcls() == null ?
- Lists.<AclEntryProto>newArrayList() :
- convertAclEntryProto(me.getAcls()))
- .addAllXAttrs(me.getxAttrs() == null ?
- Lists.<XAttrProto>newArrayList() :
- convertXAttrProto(me.getxAttrs()))
- .setXAttrsRemoved(me.isxAttrsRemoved());
- if (me.getPerms() != null) {
- metaB.setPerms(convert(me.getPerms()));
- }
- events.add(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_METADATA)
- .setContents(metaB.build().toByteString())
- .build());
- break;
- case RENAME:
- Event.RenameEvent re = (Event.RenameEvent) e;
- events.add(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_RENAME)
- .setContents(
- InotifyProtos.RenameEventProto.newBuilder()
- .setSrcPath(re.getSrcPath())
- .setDestPath(re.getDstPath())
- .setTimestamp(re.getTimestamp()).build().toByteString()
- ).build());
- break;
- case APPEND:
- Event.AppendEvent re2 = (Event.AppendEvent)
<TRUNCATED>
[27/32] hadoop git commit: YARN-1651. CapacityScheduler side changes
to support container resize. Contributed by Wangda Tan
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index ef35093..4a815f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -144,11 +144,11 @@ public class TestParentQueue {
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
((ParentQueue)queue).allocateResource(clusterResource,
- allocatedResource, RMNodeLabelsManager.NO_LABEL);
+ allocatedResource, RMNodeLabelsManager.NO_LABEL, false);
} else {
FiCaSchedulerApp app1 = getMockApplication(0, "");
((LeafQueue)queue).allocateResource(clusterResource, app1,
- allocatedResource, null, null);
+ allocatedResource, null, null, false);
}
// Next call - nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 6a0b11b..884de2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -60,6 +60,9 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;
@@ -482,6 +485,8 @@ public class TestReservations {
@Test
public void testAssignContainersNeedToUnreserve() throws Exception {
// Test that we now unreserve and use a node that has space
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
setup(csConf);
@@ -593,7 +598,7 @@ public class TestReservations {
assertEquals(2, app_0.getTotalRequiredResources(priorityReduce));
// could allocate but told need to unreserve first
- CSAssignment csAssignment = a.assignContainers(clusterResource, node_1,
+ a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
assertEquals(13 * GB, a.getUsedResources().getMemory());
assertEquals(13 * GB, app_0.getCurrentConsumption().getMemory());
@@ -664,7 +669,7 @@ public class TestReservations {
// no reserved containers - reserve then unreserve
app_0.reserve(node_0, priorityMap, rmContainer_1, container_1);
- app_0.unreserve(node_0, priorityMap);
+ app_0.unreserve(priorityMap, node_0, rmContainer_1);
unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability,
cs.getResourceCalculator(), clusterResource);
assertEquals(null, unreserveId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index dd7ed41..daccead 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -169,7 +169,7 @@ public class FairSchedulerTestBase {
resourceManager.getRMContext().getRMApps()
.put(id.getApplicationId(), rmApp);
- scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null, null, null);
return id;
}
@@ -195,7 +195,7 @@ public class FairSchedulerTestBase {
resourceManager.getRMContext().getRMApps()
.put(id.getApplicationId(), rmApp);
- scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null, null, null);
return id;
}
@@ -217,7 +217,7 @@ public class FairSchedulerTestBase {
ResourceRequest request, ApplicationAttemptId attId) {
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ask.add(request);
- scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null, null, null);
}
protected void createApplicationWithAMResource(ApplicationAttemptId attId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 53382de..65c80a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -99,7 +99,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
List<ResourceRequest> ask = new ArrayList<>();
ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true));
scheduler.allocate(
- appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+ appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);
// Advance time and let continuous scheduling kick in
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ad54616..6248e09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -1456,7 +1456,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ResourceRequest request1 =
createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true);
ask1.add(request1);
- scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null, null, null);
// Second ask, queue2 requests 1 large + (2 * minReqSize)
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
@@ -1466,14 +1466,14 @@ public class TestFairScheduler extends FairSchedulerTestBase {
false);
ask2.add(request2);
ask2.add(request3);
- scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null, null, null);
// Third ask, queue2 requests 1 large
List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
ResourceRequest request4 =
createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true);
ask3.add(request4);
- scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(), null, null, null, null);
scheduler.update();
@@ -2795,7 +2795,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Complete container
scheduler.allocate(attId, new ArrayList<ResourceRequest>(),
- Arrays.asList(containerId), null, null);
+ Arrays.asList(containerId), null, null, null, null);
assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
@@ -2887,7 +2887,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2, true));
scheduler.allocate(attemptId, asks, new ArrayList<ContainerId>(), null,
- null);
+ null, null, null);
// node 1 checks in
scheduler.update();
@@ -3283,7 +3283,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
createResourceRequest(1024, node1.getHostName(), 1, 0, true),
createResourceRequest(1024, "rack1", 1, 0, true),
createResourceRequest(1024, ResourceRequest.ANY, 1, 1, true));
- scheduler.allocate(attId1, update, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(attId1, update, new ArrayList<ContainerId>(), null, null, null, null);
// then node2 should get the container
scheduler.handle(node2UpdateEvent);
@@ -3330,7 +3330,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
anyRequest = createResourceRequest(1024, ResourceRequest.ANY,
1, 1, false);
scheduler.allocate(attId, Arrays.asList(rackRequest, anyRequest),
- new ArrayList<ContainerId>(), null, null);
+ new ArrayList<ContainerId>(), null, null, null, null);
scheduler.handle(nodeUpdateEvent);
assertEquals(0, app.getReservedContainers().size());
@@ -4332,7 +4332,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ResourceRequest request =
createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
ask.add(request);
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
// waiting for continuous_scheduler_sleep_time
// at least one pass
@@ -4352,7 +4352,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ask.clear();
ask.add(request);
scheduler.stop();
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
scheduler.continuousSchedulingAttempt();
Assert.assertEquals(2048, app.getCurrentConsumption().getMemory());
Assert.assertEquals(2, app.getCurrentConsumption().getVirtualCores());
@@ -4452,7 +4452,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ask1.add(request1);
scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null,
- null);
+ null, null, null);
String hostName = "127.0.0.1";
RMNode node1 = MockNodes.newNodeInfo(1,
@@ -4584,7 +4584,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
List<Container> containers = scheduler.allocate(appAttemptId,
Collections.<ResourceRequest> emptyList(),
- Collections.<ContainerId> emptyList(), null, null).getContainers();
+ Collections.<ContainerId> emptyList(), null, null, null, null).getContainers();
// Now with updated ResourceRequest, a container is allocated for AM.
Assert.assertTrue(containers.size() == 1);
@@ -4613,11 +4613,11 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify the blacklist can be updated independent of requesting containers
scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(),
- Collections.singletonList(host), null);
+ Collections.singletonList(host), null, null, null);
assertTrue(app.isBlacklisted(host));
scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
- Collections.singletonList(host));
+ Collections.singletonList(host), null, null);
assertFalse(scheduler.getSchedulerApp(appAttemptId).isBlacklisted(host));
List<ResourceRequest> update = Arrays.asList(
@@ -4626,7 +4626,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify a container does not actually get placed on the blacklisted host
scheduler.allocate(appAttemptId, update,
Collections.<ContainerId>emptyList(),
- Collections.singletonList(host), null);
+ Collections.singletonList(host), null, null, null);
assertTrue(app.isBlacklisted(host));
scheduler.update();
scheduler.handle(updateEvent);
@@ -4636,7 +4636,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify a container gets placed on the empty blacklist
scheduler.allocate(appAttemptId, update,
Collections.<ContainerId>emptyList(), null,
- Collections.singletonList(host));
+ Collections.singletonList(host), null, null);
assertFalse(app.isBlacklisted(host));
createSchedulingRequest(GB, "root.default", "user", 1);
scheduler.update();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 1353bdd..83ba2d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -263,7 +263,7 @@ public class TestFifoScheduler {
ask.add(nodeLocal);
ask.add(rackLocal);
ask.add(any);
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
@@ -365,7 +365,7 @@ public class TestFifoScheduler {
ask.add(nodeLocal);
ask.add(rackLocal);
ask.add(any);
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+ scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, null, null);
// Before the node update event, there are one local request
Assert.assertEquals(1, nodeLocal.getNumContainers());
@@ -941,7 +941,7 @@ public class TestFifoScheduler {
ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
fs.allocate(appAttemptId1, ask1, emptyId,
- Collections.singletonList(host_1_0), null);
+ Collections.singletonList(host_1_0), null, null, null);
// Trigger container assignment
fs.handle(new NodeUpdateSchedulerEvent(n3));
@@ -949,14 +949,14 @@ public class TestFifoScheduler {
// Get the allocation for the application and verify no allocation on
// blacklist node
Allocation allocation1 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("allocation1", 0, allocation1.getContainers().size());
// verify host_1_1 can get allocated as not in blacklist
fs.handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation2 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("allocation2", 1, allocation2.getContainers().size());
List<Container> containerList = allocation2.getContainers();
for (Container container : containerList) {
@@ -971,29 +971,29 @@ public class TestFifoScheduler {
ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
fs.allocate(appAttemptId1, ask2, emptyId,
- Collections.singletonList("rack0"), null);
+ Collections.singletonList("rack0"), null, null, null);
// verify n1 is not qualified to be allocated
fs.handle(new NodeUpdateSchedulerEvent(n1));
Allocation allocation3 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("allocation3", 0, allocation3.getContainers().size());
// verify n2 is not qualified to be allocated
fs.handle(new NodeUpdateSchedulerEvent(n2));
Allocation allocation4 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("allocation4", 0, allocation4.getContainers().size());
// verify n3 is not qualified to be allocated
fs.handle(new NodeUpdateSchedulerEvent(n3));
Allocation allocation5 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("allocation5", 0, allocation5.getContainers().size());
fs.handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation6 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("allocation6", 1, allocation6.getContainers().size());
containerList = allocation6.getContainers();
@@ -1052,25 +1052,25 @@ public class TestFifoScheduler {
List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
- fs.allocate(appAttemptId1, ask1, emptyId, null, null);
+ fs.allocate(appAttemptId1, ask1, emptyId, null, null, null, null);
// Ask for a 2 GB container for app 2
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
- fs.allocate(appAttemptId2, ask2, emptyId, null, null);
+ fs.allocate(appAttemptId2, ask2, emptyId, null, null, null, null);
// Trigger container assignment
fs.handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
Allocation allocation1 =
- fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("Allocation headroom", 1 * GB, allocation1
.getResourceLimit().getMemory());
Allocation allocation2 =
- fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
+ fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null, null, null);
Assert.assertEquals("Allocation headroom", 1 * GB, allocation2
.getResourceLimit().getMemory());
[11/32] hadoop git commit: HDFS-9111. Move hdfs-client protobuf
convert methods from PBHelper to PBHelperClient. Contributed by Mingliang
Liu.
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
index 6d9dc23..740b5cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.AcceptRecoveryRequestProto;
@@ -161,7 +162,7 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface,
.setSegmentTxnId(segmentTxId)
.setFirstTxnId(firstTxnId)
.setNumTxns(numTxns)
- .setRecords(PBHelper.getByteString(records))
+ .setRecords(PBHelperClient.getByteString(records))
.build();
try {
rpcProxy.journal(NULL_CONTROLLER, req);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
index e09ba32..3559065 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
@@ -65,7 +65,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -1048,7 +1048,7 @@ public final class CacheManager {
Expiration expiry = info.getExpiration();
if (expiry != null) {
assert (!expiry.isRelative());
- b.setExpiration(PBHelper.convert(expiry));
+ b.setExpiration(PBHelperClient.convert(expiry));
}
directives.add(b.build());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 7c3c895..0663b8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -305,7 +305,7 @@ public class EncryptionZoneManager {
}
final HdfsProtos.ZoneEncryptionInfoProto proto =
- PBHelper.convert(suite, version, keyName);
+ PBHelperClient.convert(suite, version, keyName);
final XAttr ezXAttr = XAttrHelper
.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, proto.toByteArray());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index ced085a..24d3360 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.security.AccessControlException;
@@ -284,7 +283,7 @@ class FSDirXAttrOp {
HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
fsd.ezManager.addEncryptionZone(inode.getId(),
PBHelperClient.convert(ezProto.getSuite()),
- PBHelper.convert(ezProto.getCryptoProtocolVersion()),
+ PBHelperClient.convert(ezProto.getCryptoProtocolVersion()),
ezProto.getKeyName());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index e25e0e0..4dc5326 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -1134,7 +1133,7 @@ public class FSDirectory implements Closeable {
xattr.getValue());
ezManager.unprotectedAddEncryptionZone(inode.getId(),
PBHelperClient.convert(ezProto.getSuite()),
- PBHelper.convert(ezProto.getCryptoProtocolVersion()),
+ PBHelperClient.convert(ezProto.getCryptoProtocolVersion()),
ezProto.getKeyName());
} catch (InvalidProtocolBufferException e) {
NameNode.LOG.warn("Error parsing protocol buffer of " +
@@ -1261,7 +1260,7 @@ public class FSDirectory implements Closeable {
throws IOException {
// Make the PB for the xattr
final HdfsProtos.PerFileEncryptionInfoProto proto =
- PBHelper.convertPerFileEncInfo(info);
+ PBHelperClient.convertPerFileEncInfo(info);
final byte[] protoBytes = proto.toByteArray();
final XAttr fileEncryptionAttr =
XAttrHelper.buildXAttr(CRYPTO_XATTR_FILE_ENCRYPTION_INFO, protoBytes);
@@ -1327,7 +1326,7 @@ public class FSDirectory implements Closeable {
HdfsProtos.PerFileEncryptionInfoProto fileProto =
HdfsProtos.PerFileEncryptionInfoProto.parseFrom(
fileXAttr.getValue());
- return PBHelper.convert(fileProto, suite, version, keyName);
+ return PBHelperClient.convert(fileProto, suite, version, keyName);
} catch (InvalidProtocolBufferException e) {
throw new IOException("Could not parse file encryption info for " +
"inode " + inode, e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 125e1cf..8a8a6e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -105,7 +105,7 @@ import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.protocol.proto.EditLogProtos.AclEditLogProto;
import org.apache.hadoop.hdfs.protocol.proto.EditLogProtos.XAttrEditLogProto;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.util.XMLUtils;
@@ -410,7 +410,7 @@ public abstract class FSEditLogOp {
return null;
}
XAttrEditLogProto proto = XAttrEditLogProto.parseDelimitedFrom(in);
- return PBHelper.convertXAttrs(proto.getXAttrsList());
+ return PBHelperClient.convertXAttrs(proto.getXAttrsList());
}
@SuppressWarnings("unchecked")
@@ -554,7 +554,7 @@ public abstract class FSEditLogOp {
if (this.opCode == OP_ADD) {
AclEditLogUtil.write(aclEntries, out);
XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
- b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.addAllXAttrs(PBHelperClient.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
@@ -1631,7 +1631,7 @@ public abstract class FSEditLogOp {
permissions.write(out);
AclEditLogUtil.write(aclEntries, out);
XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
- b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.addAllXAttrs(PBHelperClient.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
}
@@ -4158,7 +4158,7 @@ public abstract class FSEditLogOp {
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
- xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
+ xAttrs = PBHelperClient.convertXAttrs(p.getXAttrsList());
readRpcIds(in, logVersion);
}
@@ -4168,7 +4168,7 @@ public abstract class FSEditLogOp {
if (src != null) {
b.setSrc(src);
}
- b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.addAllXAttrs(PBHelperClient.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
// clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
@@ -4211,7 +4211,7 @@ public abstract class FSEditLogOp {
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
- xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
+ xAttrs = PBHelperClient.convertXAttrs(p.getXAttrsList());
readRpcIds(in, logVersion);
}
@@ -4221,7 +4221,7 @@ public abstract class FSEditLogOp {
if (src != null) {
b.setSrc(src);
}
- b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.addAllXAttrs(PBHelperClient.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
// clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
@@ -4267,7 +4267,7 @@ public abstract class FSEditLogOp {
throw new IOException("Failed to read fields from SetAclOp");
}
src = p.getSrc();
- aclEntries = PBHelper.convertAclEntry(p.getEntriesList());
+ aclEntries = PBHelperClient.convertAclEntry(p.getEntriesList());
}
@Override
@@ -4275,7 +4275,7 @@ public abstract class FSEditLogOp {
AclEditLogProto.Builder b = AclEditLogProto.newBuilder();
if (src != null)
b.setSrc(src);
- b.addAllEntries(PBHelper.convertAclEntryProto(aclEntries));
+ b.addAllEntries(PBHelperClient.convertAclEntryProto(aclEntries));
b.build().writeDelimitedTo(out);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index ac88919..0ae739c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -337,7 +336,7 @@ public final class FSImageFormatPBINode {
BlockInfo[] blocks = new BlockInfo[bp.size()];
for (int i = 0, e = bp.size(); i < e; ++i) {
blocks[i] =
- new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+ new BlockInfoContiguous(PBHelperClient.convert(bp.get(i)), replication);
}
final PermissionStatus permissions = loadPermission(f.getPermission(),
parent.getLoaderContext().getStringTable());
@@ -447,7 +446,7 @@ public final class FSImageFormatPBINode {
XATTR_NAMESPACE_EXT_OFFSET);
xAttrCompactBuilder.setName(v);
if (a.getValue() != null) {
- xAttrCompactBuilder.setValue(PBHelper.getByteString(a.getValue()));
+ xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue()));
}
b.addXAttrs(xAttrCompactBuilder.build());
}
@@ -636,7 +635,7 @@ public final class FSImageFormatPBINode {
if (n.getBlocks() != null) {
for (Block block : n.getBlocks()) {
- b.addBlocks(PBHelper.convert(block));
+ b.addBlocks(PBHelperClient.convert(block));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 91ebaaf..cf21411 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -244,7 +244,7 @@ public class FSImageFormatPBSnapshot {
List<BlockProto> bpl = pbf.getBlocksList();
BlockInfo[] blocks = new BlockInfo[bpl.size()];
for(int j = 0, e = bpl.size(); j < e; ++j) {
- Block blk = PBHelper.convert(bpl.get(j));
+ Block blk = PBHelperClient.convert(bpl.get(j));
BlockInfo storedBlock = bm.getStoredBlock(blk);
if(storedBlock == null) {
storedBlock = bm.addBlockCollection(
@@ -524,7 +524,7 @@ public class FSImageFormatPBSnapshot {
.setFileSize(diff.getFileSize());
if(diff.getBlocks() != null) {
for(Block block : diff.getBlocks()) {
- fb.addBlocks(PBHelper.convert(block));
+ fb.addBlocks(PBHelperClient.convert(block));
}
}
INodeFileAttributes copy = diff.snapshotINode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06022b8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 3d2e8b9..851e5b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -154,7 +154,7 @@ public class TestPBHelper {
public void testConvertDatanodeID() {
DatanodeID dn = DFSTestUtil.getLocalDatanodeID();
DatanodeIDProto dnProto = PBHelperClient.convert(dn);
- DatanodeID dn2 = PBHelper.convert(dnProto);
+ DatanodeID dn2 = PBHelperClient.convert(dnProto);
compare(dn, dn2);
}
@@ -176,8 +176,8 @@ public class TestPBHelper {
@Test
public void testConvertBlock() {
Block b = new Block(1, 100, 3);
- BlockProto bProto = PBHelper.convert(b);
- Block b2 = PBHelper.convert(bProto);
+ BlockProto bProto = PBHelperClient.convert(b);
+ Block b2 = PBHelperClient.convert(bProto);
assertEquals(b, b2);
}
@@ -399,7 +399,7 @@ public class TestPBHelper {
"identifier".getBytes(), "password".getBytes(), new Text("kind"),
new Text("service"));
TokenProto tokenProto = PBHelperClient.convert(token);
- Token<BlockTokenIdentifier> token2 = PBHelper.convert(tokenProto);
+ Token<BlockTokenIdentifier> token2 = PBHelperClient.convert(tokenProto);
compare(token, token2);
}
@@ -489,16 +489,16 @@ public class TestPBHelper {
@Test
public void testConvertLocatedBlock() {
LocatedBlock lb = createLocatedBlock();
- LocatedBlockProto lbProto = PBHelper.convert(lb);
- LocatedBlock lb2 = PBHelper.convert(lbProto);
+ LocatedBlockProto lbProto = PBHelperClient.convert(lb);
+ LocatedBlock lb2 = PBHelperClient.convert(lbProto);
compare(lb,lb2);
}
@Test
public void testConvertLocatedBlockNoStorageMedia() {
LocatedBlock lb = createLocatedBlockNoStorageMedia();
- LocatedBlockProto lbProto = PBHelper.convert(lb);
- LocatedBlock lb2 = PBHelper.convert(lbProto);
+ LocatedBlockProto lbProto = PBHelperClient.convert(lb);
+ LocatedBlock lb2 = PBHelperClient.convert(lbProto);
compare(lb,lb2);
}
@@ -508,8 +508,8 @@ public class TestPBHelper {
for (int i=0;i<3;i++) {
lbl.add(createLocatedBlock());
}
- List<LocatedBlockProto> lbpl = PBHelper.convertLocatedBlock2(lbl);
- List<LocatedBlock> lbl2 = PBHelper.convertLocatedBlock(lbpl);
+ List<LocatedBlockProto> lbpl = PBHelperClient.convertLocatedBlock2(lbl);
+ List<LocatedBlock> lbl2 = PBHelperClient.convertLocatedBlock(lbpl);
assertEquals(lbl.size(), lbl2.size());
for (int i=0;i<lbl.size();i++) {
compare(lbl.get(i), lbl2.get(2));
@@ -522,8 +522,8 @@ public class TestPBHelper {
for (int i=0;i<3;i++) {
lbl[i] = createLocatedBlock();
}
- LocatedBlockProto [] lbpl = PBHelper.convertLocatedBlock(lbl);
- LocatedBlock [] lbl2 = PBHelper.convertLocatedBlock(lbpl);
+ LocatedBlockProto [] lbpl = PBHelperClient.convertLocatedBlock(lbl);
+ LocatedBlock [] lbl2 = PBHelperClient.convertLocatedBlock(lbpl);
assertEquals(lbl.length, lbl2.length);
for (int i=0;i<lbl.length;i++) {
compare(lbl[i], lbl2[i]);
@@ -551,8 +551,8 @@ public class TestPBHelper {
DatanodeStorage dns1 = new DatanodeStorage(
"id1", DatanodeStorage.State.NORMAL, StorageType.SSD);
- DatanodeStorageProto proto = PBHelper.convert(dns1);
- DatanodeStorage dns2 = PBHelper.convert(proto);
+ DatanodeStorageProto proto = PBHelperClient.convert(dns1);
+ DatanodeStorage dns2 = PBHelperClient.convert(proto);
compare(dns1, dns2);
}
@@ -625,7 +625,7 @@ public class TestPBHelper {
.setPermission(FsAction.NONE)
.build() };
AclEntry[] actual = Lists.newArrayList(
- PBHelper.convertAclEntry(PBHelper.convertAclEntryProto(Lists
+ PBHelperClient.convertAclEntry(PBHelperClient.convertAclEntryProto(Lists
.newArrayList(e1, e2, e3)))).toArray(new AclEntry[0]);
Assert.assertArrayEquals(expected, actual);
}
@@ -637,6 +637,6 @@ public class TestPBHelper {
.setType(AclEntryType.OTHER).build();
AclStatus s = new AclStatus.Builder().owner("foo").group("bar").addEntry(e)
.build();
- Assert.assertEquals(s, PBHelper.convert(PBHelper.convert(s)));
+ Assert.assertEquals(s, PBHelperClient.convert(PBHelperClient.convert(s)));
}
}
[16/32] hadoop git commit: HDFS-8780. Fetching live/dead datanode
list with arg true for removeDecommissionNode,
returns list with decom node. (Contributed by J.Andreina)
Posted by wa...@apache.org.
HDFS-8780. Fetching live/dead datanode list with arg true for removeDecommissionNode,returns list with decom node. (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10ab7d59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10ab7d59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10ab7d59
Branch: refs/heads/YARN-1197
Commit: 10ab7d595ece59f2d00b406ba8812c6295a4187f
Parents: dfd807a
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Sep 22 12:25:35 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Sep 22 12:25:35 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/blockmanagement/DatanodeManager.java | 45 ++++----------------
.../hdfs/server/namenode/FSNamesystem.java | 8 ++--
.../apache/hadoop/hdfs/TestDecommission.java | 43 +++++++++++++++++++
4 files changed, 58 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10ab7d59/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 44774a7..af7118a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1378,6 +1378,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9063. Correctly handle snapshot path for getContentSummary. (jing9)
+ HDFS-8780. Fetching live/dead datanode list with arg true for remove-
+ DecommissionNode,returns list with decom node. (J.Andreina via vinayakumab)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10ab7d59/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 3114937..a484fcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -788,45 +788,16 @@ public class DatanodeManager {
}
/**
- * Remove an already decommissioned data node who is neither in include nor
- * exclude hosts lists from the the list of live or dead nodes. This is used
- * to not display an already decommssioned data node to the operators.
- * The operation procedure of making a already decommissioned data node not
- * to be displayed is as following:
- * <ol>
- * <li>
- * Host must have been in the include hosts list and the include hosts list
- * must not be empty.
- * </li>
- * <li>
- * Host is decommissioned by remaining in the include hosts list and added
- * into the exclude hosts list. Name node is updated with the new
- * information by issuing dfsadmin -refreshNodes command.
- * </li>
- * <li>
- * Host is removed from both include hosts and exclude hosts lists. Name
- * node is updated with the new informationby issuing dfsamin -refreshNodes
- * command.
- * <li>
- * </ol>
- *
- * @param nodeList
- * , array list of live or dead nodes.
+ * Remove decommissioned datanode from the the list of live or dead nodes.
+ * This is used to not to display a decommissioned datanode to the operators.
+ * @param nodeList , array list of live or dead nodes.
*/
- private void removeDecomNodeFromList(final List<DatanodeDescriptor> nodeList) {
- // If the include list is empty, any nodes are welcomed and it does not
- // make sense to exclude any nodes from the cluster. Therefore, no remove.
- if (!hostFileManager.hasIncludes()) {
- return;
- }
-
- for (Iterator<DatanodeDescriptor> it = nodeList.iterator(); it.hasNext();) {
+ private void removeDecomNodeFromList(
+ final List<DatanodeDescriptor> nodeList) {
+ Iterator<DatanodeDescriptor> it=null;
+ for (it = nodeList.iterator(); it.hasNext();) {
DatanodeDescriptor node = it.next();
- if ((!hostFileManager.isIncluded(node)) && (!hostFileManager.isExcluded(node))
- && node.isDecommissioned()) {
- // Include list is not empty, an existing datanode does not appear
- // in both include or exclude lists and it has been decommissioned.
- // Remove it from the node list.
+ if (node.isDecommissioned()) {
it.remove();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10ab7d59/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 75b6be9..4a9d13b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5073,7 +5073,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@Override // FSNamesystemMBean
public int getVolumeFailuresTotal() {
List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
- getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true);
+ getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false);
int volumeFailuresTotal = 0;
for (DatanodeDescriptor node: live) {
volumeFailuresTotal += node.getVolumeFailures();
@@ -5084,7 +5084,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@Override // FSNamesystemMBean
public long getEstimatedCapacityLostTotal() {
List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
- getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true);
+ getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false);
long estimatedCapacityLostTotal = 0;
for (DatanodeDescriptor node: live) {
VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary();
@@ -5891,7 +5891,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final Map<String, Map<String,Object>> info =
new HashMap<String, Map<String,Object>>();
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
- blockManager.getDatanodeManager().fetchDatanodes(live, null, true);
+ blockManager.getDatanodeManager().fetchDatanodes(live, null, false);
for (DatanodeDescriptor node : live) {
ImmutableMap.Builder<String, Object> innerinfo =
ImmutableMap.<String,Object>builder();
@@ -5939,7 +5939,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final Map<String, Map<String, Object>> info =
new HashMap<String, Map<String, Object>>();
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
- blockManager.getDatanodeManager().fetchDatanodes(null, dead, true);
+ blockManager.getDatanodeManager().fetchDatanodes(null, dead, false);
for (DatanodeDescriptor node : dead) {
Map<String, Object> innerinfo = ImmutableMap.<String, Object>builder()
.put("lastContact", getLastContact(node))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10ab7d59/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 7c30361..c1fdd25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -1129,6 +1129,49 @@ public class TestDecommission {
}
/**
+ * Fetching Live DataNodes by passing removeDecommissionedNode value as
+ * false- returns LiveNodeList with Node in Decommissioned state
+ * true - returns LiveNodeList without Node in Decommissioned state
+ * @throws InterruptedException
+ */
+ @Test
+ public void testCountOnDecommissionedNodeList() throws IOException{
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+ try {
+ cluster =
+ new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(1))
+ .numDataNodes(1).build();
+ cluster.waitActive();
+ DFSClient client = getDfsClient(cluster.getNameNode(0), conf);
+ validateCluster(client, 1);
+
+ ArrayList<ArrayList<DatanodeInfo>> namenodeDecomList =
+ new ArrayList<ArrayList<DatanodeInfo>>(1);
+ namenodeDecomList.add(0, new ArrayList<DatanodeInfo>(1));
+
+ // Move datanode1 to Decommissioned state
+ ArrayList<DatanodeInfo> decommissionedNode = namenodeDecomList.get(0);
+ decommissionNode(0, null,
+ decommissionedNode, AdminStates.DECOMMISSIONED);
+
+ FSNamesystem ns = cluster.getNamesystem(0);
+ DatanodeManager datanodeManager =
+ ns.getBlockManager().getDatanodeManager();
+ List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
+ // fetchDatanode with false should return livedecommisioned node
+ datanodeManager.fetchDatanodes(live, null, false);
+ assertTrue(1==live.size());
+ // fetchDatanode with true should not return livedecommisioned node
+ datanodeManager.fetchDatanodes(live, null, true);
+ assertTrue(0==live.size());
+ }finally {
+ cluster.shutdown();
+ }
+ }
+
+ /**
* Decommissioned node should not be considered while calculating node usage
* @throws InterruptedException
*/
[17/32] hadoop git commit: HDFS-9043. Doc updation for commands in
HDFS Federation (Contributed by J.Andreina)
Posted by wa...@apache.org.
HDFS-9043. Doc updation for commands in HDFS Federation (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57003fa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57003fa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57003fa9
Branch: refs/heads/YARN-1197
Commit: 57003fa971658c8482240f70445a6822c7692844
Parents: 10ab7d5
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Sep 22 16:27:24 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Sep 22 16:27:24 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57003fa9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index af7118a..cf54cd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1425,6 +1425,9 @@ Release 2.7.2 - UNRELEASED
HDFS-9042. Update document for the Storage policy name
(J.Andreina via vinayakumarb)
+ HDFS-9043. Doc updation for commands in HDFS Federation
+ (J.Andreina via vinayakumab)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57003fa9/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
index c00e544..38c1070 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
@@ -219,7 +219,7 @@ The policy parameter can be any of the following:
level which also balances at the Datanode level.
Note that Balancer only balances the data and does not balance the namespace.
-For the complete command usage, see [balancer](../hadoop-common/CommandsManual.html#balancer).
+For the complete command usage, see [balancer](./HDFSCommands.html#balancer).
### Decommissioning
[24/32] hadoop git commit: YARN-1643. Make ContainersMonitor support
changing monitoring size of an allocated container. Contributed by Meng Ding
and Wangda Tan
Posted by wa...@apache.org.
YARN-1643. Make ContainersMonitor support changing monitoring size of an allocated container. Contributed by Meng Ding and Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de9bb82c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de9bb82c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de9bb82c
Branch: refs/heads/YARN-1197
Commit: de9bb82cf056477f7e00ddb97fe0821cabfebd2b
Parents: 4173608
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 5 15:19:33 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:28 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../monitor/ContainersMonitorImpl.java | 207 ++++++++++------
.../TestContainerManagerWithLCE.java | 11 +
.../containermanager/TestContainerManager.java | 96 +++++++
.../monitor/MockResourceCalculatorPlugin.java | 69 ++++++
.../MockResourceCalculatorProcessTree.java | 57 +++++
.../TestContainersMonitorResourceChange.java | 248 +++++++++++++++++++
7 files changed, 615 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cb3daad..acff3d6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -212,6 +212,9 @@ Release 2.8.0 - UNRELEASED
YARN-3867. ContainerImpl changes to support container resizing. (Meng Ding
via jianhe)
+ YARN-1643. Make ContainersMonitor support changing monitoring size of an
+ allocated container. (Meng Ding and Wangda Tan)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index afb51ad..b3839d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,12 +30,14 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
@@ -56,16 +56,16 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean containerMetricsEnabled;
private long containerMetricsPeriodMs;
- final List<ContainerId> containersToBeRemoved;
- final Map<ContainerId, ProcessTreeInfo> containersToBeAdded;
- Map<ContainerId, ProcessTreeInfo> trackingContainers =
- new HashMap<ContainerId, ProcessTreeInfo>();
+ @VisibleForTesting
+ final Map<ContainerId, ProcessTreeInfo> trackingContainers =
+ new ConcurrentHashMap<>();
- final ContainerExecutor containerExecutor;
+ private final ContainerExecutor containerExecutor;
private final Dispatcher eventDispatcher;
private final Context context;
private ResourceCalculatorPlugin resourceCalculatorPlugin;
private Configuration conf;
+ private static float vmemRatio;
private Class<? extends ResourceCalculatorProcessTree> processTreeClass;
private long maxVmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT;
@@ -82,6 +82,8 @@ public class ContainersMonitorImpl extends AbstractService implements
private ResourceUtilization containersUtilization;
+ private volatile boolean stopped = false;
+
public ContainersMonitorImpl(ContainerExecutor exec,
AsyncDispatcher dispatcher, Context context) {
super("containers-monitor");
@@ -90,8 +92,6 @@ public class ContainersMonitorImpl extends AbstractService implements
this.eventDispatcher = dispatcher;
this.context = context;
- this.containersToBeAdded = new HashMap<ContainerId, ProcessTreeInfo>();
- this.containersToBeRemoved = new ArrayList<ContainerId>();
this.monitoringThread = new MonitoringThread();
this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
@@ -140,7 +140,7 @@ public class ContainersMonitorImpl extends AbstractService implements
this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
// ///////// Virtual memory configuration //////
- float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
Preconditions.checkArgument(vmemRatio > 0.99f,
YarnConfiguration.NM_VMEM_PMEM_RATIO + " should be at least 1.0");
@@ -218,6 +218,7 @@ public class ContainersMonitorImpl extends AbstractService implements
@Override
protected void serviceStop() throws Exception {
if (containersMonitorEnabled) {
+ stopped = true;
this.monitoringThread.interrupt();
try {
this.monitoringThread.join();
@@ -228,7 +229,8 @@ public class ContainersMonitorImpl extends AbstractService implements
super.serviceStop();
}
- private static class ProcessTreeInfo {
+ @VisibleForTesting
+ static class ProcessTreeInfo {
private ContainerId containerId;
private String pid;
private ResourceCalculatorProcessTree pTree;
@@ -267,26 +269,43 @@ public class ContainersMonitorImpl extends AbstractService implements
this.pTree = pTree;
}
- public long getVmemLimit() {
+ /**
+ * @return Virtual memory limit for the process tree in bytes
+ */
+ public synchronized long getVmemLimit() {
return this.vmemLimit;
}
/**
* @return Physical memory limit for the process tree in bytes
*/
- public long getPmemLimit() {
+ public synchronized long getPmemLimit() {
return this.pmemLimit;
}
/**
- * Return the number of cpu vcores assigned
- * @return
+ * @return Number of cpu vcores assigned
*/
- public int getCpuVcores() {
+ public synchronized int getCpuVcores() {
return this.cpuVcores;
}
- }
+ /**
+ * Set resource limit for enforcement
+ * @param pmemLimit
+ * Physical memory limit for the process tree in bytes
+ * @param vmemLimit
+ * Virtual memory limit for the process tree in bytes
+ * @param cpuVcores
+ * Number of cpu vcores assigned
+ */
+ public synchronized void setResourceLimit(
+ long pmemLimit, long vmemLimit, int cpuVcores) {
+ this.pmemLimit = pmemLimit;
+ this.vmemLimit = vmemLimit;
+ this.cpuVcores = cpuVcores;
+ }
+ }
/**
* Check whether a container's process tree's current memory usage is over
@@ -359,8 +378,7 @@ public class ContainersMonitorImpl extends AbstractService implements
@Override
public void run() {
- while (true) {
-
+ while (!stopped && !Thread.currentThread().isInterrupted()) {
// Print the processTrees for debugging.
if (LOG.isDebugEnabled()) {
StringBuilder tmp = new StringBuilder("[ ");
@@ -372,31 +390,6 @@ public class ContainersMonitorImpl extends AbstractService implements
+ tmp.substring(0, tmp.length()) + "]");
}
- // Add new containers
- synchronized (containersToBeAdded) {
- for (Entry<ContainerId, ProcessTreeInfo> entry : containersToBeAdded
- .entrySet()) {
- ContainerId containerId = entry.getKey();
- ProcessTreeInfo processTreeInfo = entry.getValue();
- LOG.info("Starting resource-monitoring for " + containerId);
- trackingContainers.put(containerId, processTreeInfo);
- }
- containersToBeAdded.clear();
- }
-
- // Remove finished containers
- synchronized (containersToBeRemoved) {
- for (ContainerId containerId : containersToBeRemoved) {
- if (containerMetricsEnabled) {
- ContainerMetrics.forContainer(
- containerId, containerMetricsPeriodMs).finished();
- }
- trackingContainers.remove(containerId);
- LOG.info("Stopping resource-monitoring for " + containerId);
- }
- containersToBeRemoved.clear();
- }
-
// Temporary structure to calculate the total resource utilization of
// the containers
ResourceUtilization trackedContainersUtilization =
@@ -408,10 +401,8 @@ public class ContainersMonitorImpl extends AbstractService implements
long pmemByAllContainers = 0;
long cpuUsagePercentPerCoreByAllContainers = 0;
long cpuUsageTotalCoresByAllContainers = 0;
- for (Iterator<Map.Entry<ContainerId, ProcessTreeInfo>> it =
- trackingContainers.entrySet().iterator(); it.hasNext();) {
-
- Map.Entry<ContainerId, ProcessTreeInfo> entry = it.next();
+ for (Entry<ContainerId, ProcessTreeInfo> entry : trackingContainers
+ .entrySet()) {
ContainerId containerId = entry.getKey();
ProcessTreeInfo ptInfo = entry.getValue();
try {
@@ -435,11 +426,6 @@ public class ContainersMonitorImpl extends AbstractService implements
if (containerMetricsEnabled) {
ContainerMetrics usageMetrics = ContainerMetrics
.forContainer(containerId, containerMetricsPeriodMs);
- int cpuVcores = ptInfo.getCpuVcores();
- final int vmemLimit = (int) (ptInfo.getVmemLimit() >> 20);
- final int pmemLimit = (int) (ptInfo.getPmemLimit() >> 20);
- usageMetrics.recordResourceLimit(
- vmemLimit, pmemLimit, cpuVcores);
usageMetrics.recordProcessId(pId);
}
}
@@ -548,7 +534,7 @@ public class ContainersMonitorImpl extends AbstractService implements
eventDispatcher.getEventHandler().handle(
new ContainerKillEvent(containerId,
containerExitStatus, msg));
- it.remove();
+ trackingContainers.remove(containerId);
LOG.info("Removed ProcessTree with root " + pId);
}
} catch (Exception e) {
@@ -605,6 +591,60 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
+ private void changeContainerResource(
+ ContainerId containerId, Resource resource) {
+ Container container = context.getContainers().get(containerId);
+ // Check container existence
+ if (container == null) {
+ LOG.warn("Container " + containerId.toString() + "does not exist");
+ return;
+ }
+ container.setResource(resource);
+ }
+
+ private void updateContainerMetrics(ContainersMonitorEvent monitoringEvent) {
+ if (!containerMetricsEnabled || monitoringEvent == null) {
+ return;
+ }
+
+ ContainerId containerId = monitoringEvent.getContainerId();
+ ContainerMetrics usageMetrics = ContainerMetrics
+ .forContainer(containerId, containerMetricsPeriodMs);
+
+ int vmemLimitMBs;
+ int pmemLimitMBs;
+ int cpuVcores;
+ switch (monitoringEvent.getType()) {
+ case START_MONITORING_CONTAINER:
+ ContainerStartMonitoringEvent startEvent =
+ (ContainerStartMonitoringEvent) monitoringEvent;
+ usageMetrics.recordStateChangeDurations(
+ startEvent.getLaunchDuration(),
+ startEvent.getLocalizationDuration());
+ cpuVcores = startEvent.getCpuVcores();
+ vmemLimitMBs = (int) (startEvent.getVmemLimit() >> 20);
+ pmemLimitMBs = (int) (startEvent.getPmemLimit() >> 20);
+ usageMetrics.recordResourceLimit(
+ vmemLimitMBs, pmemLimitMBs, cpuVcores);
+ break;
+ case STOP_MONITORING_CONTAINER:
+ usageMetrics.finished();
+ break;
+ case CHANGE_MONITORING_CONTAINER_RESOURCE:
+ ChangeMonitoringContainerResourceEvent changeEvent =
+ (ChangeMonitoringContainerResourceEvent) monitoringEvent;
+ Resource resource = changeEvent.getResource();
+ pmemLimitMBs = resource.getMemory();
+ vmemLimitMBs = (int) (pmemLimitMBs * vmemRatio);
+ cpuVcores = resource.getVirtualCores();
+ usageMetrics.recordResourceLimit(
+ vmemLimitMBs, pmemLimitMBs, cpuVcores);
+ break;
+ default:
+ break;
+ }
+ }
+
@Override
public long getVmemAllocatedForContainers() {
return this.maxVmemAllottedForContainers;
@@ -650,38 +690,53 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
+ @SuppressWarnings("unchecked")
public void handle(ContainersMonitorEvent monitoringEvent) {
-
+ ContainerId containerId = monitoringEvent.getContainerId();
if (!containersMonitorEnabled) {
+ if (monitoringEvent.getType() == ContainersMonitorEventType
+ .CHANGE_MONITORING_CONTAINER_RESOURCE) {
+ // Nothing to enforce. Update container resource immediately.
+ ChangeMonitoringContainerResourceEvent changeEvent =
+ (ChangeMonitoringContainerResourceEvent) monitoringEvent;
+ changeContainerResource(containerId, changeEvent.getResource());
+ }
return;
}
- ContainerId containerId = monitoringEvent.getContainerId();
switch (monitoringEvent.getType()) {
case START_MONITORING_CONTAINER:
ContainerStartMonitoringEvent startEvent =
(ContainerStartMonitoringEvent) monitoringEvent;
-
- if (containerMetricsEnabled) {
- ContainerMetrics usageMetrics = ContainerMetrics
- .forContainer(containerId, containerMetricsPeriodMs);
- usageMetrics.recordStateChangeDurations(
- startEvent.getLaunchDuration(),
- startEvent.getLocalizationDuration());
- }
-
- synchronized (this.containersToBeAdded) {
- ProcessTreeInfo processTreeInfo =
- new ProcessTreeInfo(containerId, null, null,
- startEvent.getVmemLimit(), startEvent.getPmemLimit(),
- startEvent.getCpuVcores());
- this.containersToBeAdded.put(containerId, processTreeInfo);
- }
+ LOG.info("Starting resource-monitoring for " + containerId);
+ updateContainerMetrics(monitoringEvent);
+ trackingContainers.put(containerId,
+ new ProcessTreeInfo(containerId, null, null,
+ startEvent.getVmemLimit(), startEvent.getPmemLimit(),
+ startEvent.getCpuVcores()));
break;
case STOP_MONITORING_CONTAINER:
- synchronized (this.containersToBeRemoved) {
- this.containersToBeRemoved.add(containerId);
+ LOG.info("Stopping resource-monitoring for " + containerId);
+ updateContainerMetrics(monitoringEvent);
+ trackingContainers.remove(containerId);
+ break;
+ case CHANGE_MONITORING_CONTAINER_RESOURCE:
+ ChangeMonitoringContainerResourceEvent changeEvent =
+ (ChangeMonitoringContainerResourceEvent) monitoringEvent;
+ ProcessTreeInfo processTreeInfo = trackingContainers.get(containerId);
+ if (processTreeInfo == null) {
+ LOG.warn("Failed to track container "
+ + containerId.toString()
+ + ". It may have already completed.");
+ break;
}
+ LOG.info("Changing resource-monitoring for " + containerId);
+ updateContainerMetrics(monitoringEvent);
+ long pmemLimit = changeEvent.getResource().getMemory() * 1024L * 1024L;
+ long vmemLimit = (long) (pmemLimit * vmemRatio);
+ int cpuVcores = changeEvent.getResource().getVirtualCores();
+ processTreeInfo.setResourceLimit(pmemLimit, vmemLimit, cpuVcores);
+ changeContainerResource(containerId, changeEvent.getResource());
break;
default:
// TODO: Wrong event.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 9a05278..75bcdae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -211,6 +211,17 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
super.testIncreaseContainerResourceWithInvalidResource();
}
+ @Override
+ public void testChangeContainerResource() throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testChangeContainerResource");
+ super.testChangeContainerResource();
+ }
+
private boolean shouldRunTest() {
return System
.getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index e2f12ba..2ea9146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -1046,6 +1046,102 @@ public class TestContainerManager extends BaseContainerManagerTest {
}
}
+ @Test
+ public void testChangeContainerResource() throws Exception {
+ containerManager.start();
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
+ PrintWriter fileWriter = new PrintWriter(scriptFile);
+ // Construct the Container-id
+ ContainerId cId = createContainerId(0);
+ if (Shell.WINDOWS) {
+ fileWriter.println("@ping -n 100 127.0.0.1 >nul");
+ } else {
+ fileWriter.write("\numask 0");
+ fileWriter.write("\nexec sleep 100");
+ }
+ fileWriter.close();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ URL resource_alpha =
+ ConverterUtils.getYarnUrlFromPath(localFS
+ .makeQualified(new Path(scriptFile.getAbsolutePath())));
+ LocalResource rsrc_alpha =
+ recordFactory.newRecordInstance(LocalResource.class);
+ rsrc_alpha.setResource(resource_alpha);
+ rsrc_alpha.setSize(-1);
+ rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+ rsrc_alpha.setType(LocalResourceType.FILE);
+ rsrc_alpha.setTimestamp(scriptFile.lastModified());
+ String destinationFile = "dest_file";
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, rsrc_alpha);
+ containerLaunchContext.setLocalResources(localResources);
+ List<String> commands =
+ Arrays.asList(Shell.getRunScriptCommand(scriptFile));
+ containerLaunchContext.setCommands(commands);
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+ // Make sure the container reaches RUNNING state
+ BaseContainerManagerTest.waitForNMContainerState(containerManager, cId,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING);
+ // Construct container resource increase request,
+ List<Token> increaseTokens = new ArrayList<Token>();
+ // Add increase request.
+ Resource targetResource = Resource.newInstance(4096, 2);
+ Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user, targetResource,
+ context.getContainerTokenSecretManager(), null);
+ increaseTokens.add(containerToken);
+ IncreaseContainersResourceRequest increaseRequest =
+ IncreaseContainersResourceRequest.newInstance(increaseTokens);
+ IncreaseContainersResourceResponse increaseResponse =
+ containerManager.increaseContainersResource(increaseRequest);
+ Assert.assertEquals(
+ 1, increaseResponse.getSuccessfullyIncreasedContainers().size());
+ Assert.assertTrue(increaseResponse.getFailedRequests().isEmpty());
+ // Check status
+ List<ContainerId> containerIds = new ArrayList<>();
+ containerIds.add(cId);
+ GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
+ ContainerStatus containerStatus = containerManager
+ .getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
+ // Check status immediately as resource increase is blocking
+ assertEquals(targetResource, containerStatus.getCapability());
+ // Simulate a decrease request
+ List<org.apache.hadoop.yarn.api.records.Container> containersToDecrease
+ = new ArrayList<>();
+ targetResource = Resource.newInstance(2048, 2);
+ org.apache.hadoop.yarn.api.records.Container decreasedContainer =
+ org.apache.hadoop.yarn.api.records.Container
+ .newInstance(cId, null, null, targetResource, null, null);
+ containersToDecrease.add(decreasedContainer);
+ containerManager.handle(
+ new CMgrDecreaseContainersResourceEvent(containersToDecrease));
+ // Check status with retry
+ containerStatus = containerManager
+ .getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
+ int retry = 0;
+ while (!targetResource.equals(containerStatus.getCapability()) &&
+ (retry++ < 5)) {
+ Thread.sleep(200);
+ containerStatus = containerManager.getContainerStatuses(gcsRequest)
+ .getContainerStatuses().get(0);
+ }
+ assertEquals(targetResource, containerStatus.getCapability());
+ }
+
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
NodeId nodeId, String user,
NMContainerTokenSecretManager containerTokenSecretManager)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
new file mode 100644
index 0000000..4a18a8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+
+public class MockResourceCalculatorPlugin extends ResourceCalculatorPlugin {
+
+ @Override
+ public long getVirtualMemorySize() {
+ return 0;
+ }
+
+ @Override
+ public long getPhysicalMemorySize() {
+ return 0;
+ }
+
+ @Override
+ public long getAvailableVirtualMemorySize() {
+ return 0;
+ }
+
+ @Override
+ public long getAvailablePhysicalMemorySize() {
+ return 0;
+ }
+
+ @Override
+ public int getNumProcessors() {
+ return 0;
+ }
+
+ @Override
+ public int getNumCores() {
+ return 0;
+ }
+
+ @Override
+ public long getCpuFrequency() {
+ return 0;
+ }
+
+ @Override
+ public long getCumulativeCpuTime() {
+ return 0;
+ }
+
+ @Override
+ public float getCpuUsage() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java
new file mode 100644
index 0000000..c5aaa77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+public class MockResourceCalculatorProcessTree extends ResourceCalculatorProcessTree {
+
+ private long rssMemorySize = 0;
+
+ public MockResourceCalculatorProcessTree(String root) {
+ super(root);
+ }
+
+ @Override
+ public void updateProcessTree() {
+ }
+
+ @Override
+ public String getProcessTreeDump() {
+ return "";
+ }
+
+ @Override
+ public long getCumulativeCpuTime() {
+ return 0;
+ }
+
+ @Override
+ public boolean checkPidPgrpidForMatch() {
+ return true;
+ }
+
+ public void setRssMemorySize(long rssMemorySize) {
+ this.rssMemorySize = rssMemorySize;
+ }
+
+ public long getRssMemorySize() {
+ return this.rssMemorySize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de9bb82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
new file mode 100644
index 0000000..d7f89fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ProcessTreeInfo;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+public class TestContainersMonitorResourceChange {
+
+ private ContainersMonitorImpl containersMonitor;
+ private MockExecutor executor;
+ private Configuration conf;
+ private AsyncDispatcher dispatcher;
+ private Context context;
+ private MockContainerEventHandler containerEventHandler;
+
+ private static class MockExecutor extends ContainerExecutor {
+ @Override
+ public void init() throws IOException {
+ }
+ @Override
+ public void startLocalizer(LocalizerStartContext ctx)
+ throws IOException, InterruptedException {
+ }
+ @Override
+ public int launchContainer(ContainerStartContext ctx) throws
+ IOException {
+ return 0;
+ }
+ @Override
+ public boolean signalContainer(ContainerSignalContext ctx)
+ throws IOException {
+ return true;
+ }
+ @Override
+ public void deleteAsUser(DeletionAsUserContext ctx)
+ throws IOException, InterruptedException {
+ }
+ @Override
+ public String getProcessId(ContainerId containerId) {
+ return String.valueOf(containerId.getContainerId());
+ }
+ @Override
+ public boolean isContainerAlive(ContainerLivenessContext ctx)
+ throws IOException {
+ return true;
+ }
+ }
+
+ private static class MockContainerEventHandler implements
+ EventHandler<ContainerEvent> {
+ final private Set<ContainerId> killedContainer
+ = new HashSet<>();
+ @Override
+ public void handle(ContainerEvent event) {
+ if (event.getType() == ContainerEventType.KILL_CONTAINER) {
+ synchronized (killedContainer) {
+ killedContainer.add(event.getContainerID());
+ }
+ }
+ }
+ public boolean isContainerKilled(ContainerId containerId) {
+ synchronized (killedContainer) {
+ return killedContainer.contains(containerId);
+ }
+ }
+ }
+
+ @Before
+ public void setup() {
+ executor = new MockExecutor();
+ dispatcher = new AsyncDispatcher();
+ context = Mockito.mock(Context.class);
+ Mockito.doReturn(new ConcurrentSkipListMap<ContainerId, Container>())
+ .when(context).getContainers();
+ conf = new Configuration();
+ conf.set(
+ YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
+ MockResourceCalculatorPlugin.class.getCanonicalName());
+ conf.set(
+ YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
+ MockResourceCalculatorProcessTree.class.getCanonicalName());
+ dispatcher.init(conf);
+ dispatcher.start();
+ containerEventHandler = new MockContainerEventHandler();
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (containersMonitor != null) {
+ containersMonitor.stop();
+ }
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+
+ @Test
+ public void testContainersResourceChange() throws Exception {
+ // set container monitor interval to be 20ms
+ conf.setLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS, 20L);
+ containersMonitor = createContainersMonitor(executor, dispatcher, context);
+ containersMonitor.init(conf);
+ containersMonitor.start();
+ // create container 1
+ containersMonitor.handle(new ContainerStartMonitoringEvent(
+ getContainerId(1), 2100L, 1000L, 1, 0, 0));
+ // verify that this container is properly tracked
+ assertNotNull(getProcessTreeInfo(getContainerId(1)));
+ assertEquals(1000L, getProcessTreeInfo(getContainerId(1))
+ .getPmemLimit());
+ assertEquals(2100L, getProcessTreeInfo(getContainerId(1))
+ .getVmemLimit());
+ // sleep longer than the monitor interval to make sure resource
+ // enforcement has started
+ Thread.sleep(200);
+ // increase pmem usage, the container should be killed
+ MockResourceCalculatorProcessTree mockTree =
+ (MockResourceCalculatorProcessTree) getProcessTreeInfo(
+ getContainerId(1)).getProcessTree();
+ mockTree.setRssMemorySize(2500L);
+ // verify that this container is killed
+ Thread.sleep(200);
+ assertTrue(containerEventHandler
+ .isContainerKilled(getContainerId(1)));
+ // create container 2
+ containersMonitor.handle(new ContainerStartMonitoringEvent(
+ getContainerId(2), 2202009L, 1048576L, 1, 0, 0));
+ // verify that this container is properly tracked
+ assertNotNull(getProcessTreeInfo(getContainerId(2)));
+ assertEquals(1048576L, getProcessTreeInfo(getContainerId(2))
+ .getPmemLimit());
+ assertEquals(2202009L, getProcessTreeInfo(getContainerId(2))
+ .getVmemLimit());
+ // trigger a change resource event, check limit after change
+ containersMonitor.handle(new ChangeMonitoringContainerResourceEvent(
+ getContainerId(2), Resource.newInstance(2, 1)));
+ assertEquals(2097152L, getProcessTreeInfo(getContainerId(2))
+ .getPmemLimit());
+ assertEquals(4404019L, getProcessTreeInfo(getContainerId(2))
+ .getVmemLimit());
+ // sleep longer than the monitor interval to make sure resource
+ // enforcement has started
+ Thread.sleep(200);
+ // increase pmem usage, the container should NOT be killed
+ mockTree =
+ (MockResourceCalculatorProcessTree) getProcessTreeInfo(
+ getContainerId(2)).getProcessTree();
+ mockTree.setRssMemorySize(2000000L);
+ // verify that this container is not killed
+ Thread.sleep(200);
+ assertFalse(containerEventHandler
+ .isContainerKilled(getContainerId(2)));
+ containersMonitor.stop();
+ }
+
+ @Test
+ public void testContainersResourceChangeIsTriggeredImmediately()
+ throws Exception {
+ // set container monitor interval to be 20s
+ conf.setLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS, 20000L);
+ containersMonitor = createContainersMonitor(executor, dispatcher, context);
+ containersMonitor.init(conf);
+ containersMonitor.start();
+ // sleep 1 second to make sure the container monitor thread is
+ // now waiting for the next monitor cycle
+ Thread.sleep(1000);
+ // create a container with id 3
+ containersMonitor.handle(new ContainerStartMonitoringEvent(
+ getContainerId(3), 2202009L, 1048576L, 1, 0, 0));
+ // Verify that this container has been tracked
+ assertNotNull(getProcessTreeInfo(getContainerId(3)));
+ // trigger a change resource event, check limit after change
+ containersMonitor.handle(new ChangeMonitoringContainerResourceEvent(
+ getContainerId(3), Resource.newInstance(2, 1)));
+ // verify that this container has been properly tracked with the
+ // correct size
+ assertEquals(2097152L, getProcessTreeInfo(getContainerId(3))
+ .getPmemLimit());
+ assertEquals(4404019L, getProcessTreeInfo(getContainerId(3))
+ .getVmemLimit());
+ containersMonitor.stop();
+ }
+
+ private ContainersMonitorImpl createContainersMonitor(
+ ContainerExecutor containerExecutor, AsyncDispatcher dispatcher,
+ Context context) {
+ return new ContainersMonitorImpl(containerExecutor, dispatcher, context);
+ }
+
+ private ContainerId getContainerId(int id) {
+ return ContainerId.newContainerId(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(123456L, 1), 1), id);
+ }
+
+ private ProcessTreeInfo getProcessTreeInfo(ContainerId id) {
+ return containersMonitor.trackingContainers.get(id);
+ }
+}
[18/32] hadoop git commit: MAPREDUCE-5045. UtilTest#isCygwin method
appears to be unused. Contributed by Neelesh Srinivas Salian.
Posted by wa...@apache.org.
MAPREDUCE-5045. UtilTest#isCygwin method appears to be unused. Contributed by Neelesh Srinivas Salian.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc2b4739
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc2b4739
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc2b4739
Branch: refs/heads/YARN-1197
Commit: cc2b4739902df60254dce2ddb23ef8f6ff2a3495
Parents: 57003fa
Author: Harsh J <ha...@cloudera.com>
Authored: Tue Sep 22 21:37:41 2015 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Tue Sep 22 21:38:06 2015 +0530
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/test/java/org/apache/hadoop/streaming/UtilTest.java | 5 -----
2 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc2b4739/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index bcdac1f..c2fe31f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -295,6 +295,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-5045. UtilTest#isCygwin method appears to be unused
+ (Neelesh Srinivas Salian via harsh)
+
MAPREDUCE-6291. Correct mapred queue usage command.
(Brahma Reddu Battula via harsh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc2b4739/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
index 2766969..31e4905 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
@@ -117,11 +117,6 @@ class UtilTest {
return collate(vargs, " ");
}
- public static boolean isCygwin() {
- String OS = System.getProperty("os.name");
- return (OS.indexOf("Windows") > -1);
- }
-
/**
* Is perl supported on this machine ?
* @return true if perl is available and is working as expected
[04/32] hadoop git commit: HDFS-9101. Remove deprecated
NameNode.getUri() static helper method. Contributed by Mingliang Liu.
Posted by wa...@apache.org.
HDFS-9101. Remove deprecated NameNode.getUri() static helper method. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66b46d08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66b46d08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66b46d08
Branch: refs/heads/YARN-1197
Commit: 66b46d0885f7049d0485e0d08b5e7af9762f0a59
Parents: 94dec5a
Author: Haohui Mai <wh...@apache.org>
Authored: Fri Sep 18 18:23:53 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Sep 18 18:23:53 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/namenode/NameNode.java | 7 -------
2 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66b46d08/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b905d42..65cde45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -935,6 +935,9 @@ Release 2.8.0 - UNRELEASED
HDFS-5802. NameNode does not check for inode type before traversing down a
path. (Xiao Chen via Yongjun Zhang)
+ HDFS-9101. Remove deprecated NameNode.getUri() static helper method.
+ (Mingliang Liu via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66b46d08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 0431fee..565555e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -471,13 +471,6 @@ public class NameNode implements NameNodeStatusMXBean {
return DFSUtilClient.getNNAddress(addr);
}
- @Deprecated
- /**
- * @deprecated Use {@link DFSUtilClient#getNNUri(InetSocketAddress)} instead.
- */
- public static URI getUri(InetSocketAddress namenode) {
- return DFSUtilClient.getNNUri(namenode);
- }
//
// Common NameNode methods implementation for the active name-node role.
//
[26/32] hadoop git commit: YARN-3868. Recovery support for container
resizing. Contributed by Meng Ding
Posted by wa...@apache.org.
YARN-3868. Recovery support for container resizing. Contributed by Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d076b7ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d076b7ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d076b7ba
Branch: refs/heads/YARN-1197
Commit: d076b7ba2464b5efa73145fd20ef8e58dd868e77
Parents: 1e4034a
Author: Jian He <ji...@apache.org>
Authored: Thu Aug 20 21:18:23 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:29 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../containermanager/ContainerManagerImpl.java | 5 +-
.../container/ContainerImpl.java | 8 +-
.../recovery/NMLeveldbStateStoreService.java | 22 ++
.../recovery/NMNullStateStoreService.java | 6 +
.../recovery/NMStateStoreService.java | 15 ++
.../TestContainerManagerRecovery.java | 233 ++++++++++++++++++-
.../recovery/NMMemoryStateStoreService.java | 11 +-
.../TestNMLeveldbStateStoreService.java | 11 +
9 files changed, 301 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 274ffc4..2f055ca 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -218,6 +218,8 @@ Release 2.8.0 - UNRELEASED
YARN-1644. RM-NM protocol changes and NodeStatusUpdater implementation to
support container resizing. (Meng Ding via jianhe)
+ YARN-3868. Recovery support for container resizing. (Meng Ding via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 868d8d3..39d2983 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -346,7 +346,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container = new ContainerImpl(getConfig(), dispatcher,
context.getNMStateStore(), req.getContainerLaunchContext(),
credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
- rcs.getDiagnostics(), rcs.getKilled());
+ rcs.getDiagnostics(), rcs.getKilled(), rcs.getCapability());
context.getContainers().put(containerId, container);
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
@@ -1101,6 +1101,9 @@ public class ContainerManagerImpl extends CompositeService implements
this.readLock.lock();
try {
if (!serviceStopped) {
+ // Persist container resource change for recovery
+ this.context.getNMStateStore().storeContainerResourceChanged(
+ containerId, targetResource);
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(
containerId, targetResource));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 5c61a92..eff2188 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -154,13 +154,19 @@ public class ContainerImpl implements Container {
Credentials creds, NodeManagerMetrics metrics,
ContainerTokenIdentifier containerTokenIdentifier,
RecoveredContainerStatus recoveredStatus, int exitCode,
- String diagnostics, boolean wasKilled) {
+ String diagnostics, boolean wasKilled, Resource recoveredCapability) {
this(conf, dispatcher, stateStore, launchContext, creds, metrics,
containerTokenIdentifier);
this.recoveredStatus = recoveredStatus;
this.exitCode = exitCode;
this.recoveredAsKilled = wasKilled;
this.diagnostics.append(diagnostics);
+ if (recoveredCapability != null
+ && !this.resource.equals(recoveredCapability)) {
+ // resource capability had been updated before NM was down
+ this.resource = Resource.newInstance(recoveredCapability.getMemory(),
+ recoveredCapability.getVirtualCores());
+ }
}
private static final ContainerDiagnosticsUpdateTransition UPDATE_DIAGNOSTICS_TRANSITION =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index df58182..89c71bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -40,7 +40,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestP
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
@@ -99,6 +102,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
private static final String CONTAINER_REQUEST_KEY_SUFFIX = "/request";
private static final String CONTAINER_DIAGS_KEY_SUFFIX = "/diagnostics";
private static final String CONTAINER_LAUNCHED_KEY_SUFFIX = "/launched";
+ private static final String CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX =
+ "/resourceChanged";
private static final String CONTAINER_KILLED_KEY_SUFFIX = "/killed";
private static final String CONTAINER_EXIT_CODE_KEY_SUFFIX = "/exitcode";
@@ -230,6 +235,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
} else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
rcs.status = RecoveredContainerStatus.COMPLETED;
rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
+ } else if (suffix.equals(CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX)) {
+ rcs.capability = new ResourcePBImpl(
+ ResourceProto.parseFrom(entry.getValue()));
} else {
throw new IOException("Unexpected container state key: " + key);
}
@@ -275,6 +283,20 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
@Override
+ public void storeContainerResourceChanged(ContainerId containerId,
+ Resource capability) throws IOException {
+ String key = CONTAINERS_KEY_PREFIX + containerId.toString()
+ + CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX;
+ try {
+ // New value will overwrite old values for the same key
+ db.put(bytes(key),
+ ((ResourcePBImpl) capability).getProto().toByteArray());
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
public void storeContainerKilled(ContainerId containerId)
throws IOException {
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
index ab49543..d5dce9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
@@ -88,6 +89,11 @@ public class NMNullStateStoreService extends NMStateStoreService {
}
@Override
+ public void storeContainerResourceChanged(ContainerId containerId,
+ Resource capability) throws IOException {
+ }
+
+ @Override
public void storeContainerKilled(ContainerId containerId)
throws IOException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
index fa66349..e8ccf54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
@@ -74,6 +75,7 @@ public abstract class NMStateStoreService extends AbstractService {
boolean killed = false;
String diagnostics = "";
StartContainerRequest startRequest;
+ Resource capability;
public RecoveredContainerStatus getStatus() {
return status;
@@ -94,6 +96,10 @@ public abstract class NMStateStoreService extends AbstractService {
public StartContainerRequest getStartRequest() {
return startRequest;
}
+
+ public Resource getCapability() {
+ return capability;
+ }
}
public static class LocalResourceTrackerState {
@@ -284,6 +290,15 @@ public abstract class NMStateStoreService extends AbstractService {
throws IOException;
/**
+ * Record that a container resource has been changed
+ * @param containerId the container ID
+ * @param capability the container resource capability
+ * @throws IOException
+ */
+ public abstract void storeContainerResourceChanged(ContainerId containerId,
+ Resource capability) throws IOException;
+
+ /**
* Record that a container has completed
* @param containerId the container ID
* @param exitCode the exit code from the container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 4d0aacd..43f1b29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -28,18 +28,30 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
@@ -48,9 +60,17 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
@@ -58,6 +78,9 @@ import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
@@ -65,6 +88,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
@@ -77,18 +101,50 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Before;
import org.junit.Test;
-public class TestContainerManagerRecovery {
+public class TestContainerManagerRecovery extends BaseContainerManagerTest {
- private NodeManagerMetrics metrics = NodeManagerMetrics.create();
+ public TestContainerManagerRecovery() throws UnsupportedFileSystemException {
+ super();
+ }
+
+ @Override
+ @Before
+ public void setup() throws IOException {
+ localFS.delete(new Path(localDir.getAbsolutePath()), true);
+ localFS.delete(new Path(tmpDir.getAbsolutePath()), true);
+ localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
+ localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
+ localDir.mkdir();
+ tmpDir.mkdir();
+ localLogDir.mkdir();
+ remoteLogDir.mkdir();
+ LOG.info("Created localDir in " + localDir.getAbsolutePath());
+ LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
+
+ String bindAddress = "0.0.0.0:12345";
+ conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
+ conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
+ // Default delSrvc
+ delSrvc = createDeletionService();
+ delSrvc.init(conf);
+ exec = createContainerExecutor();
+ dirsHandler = new LocalDirsHandlerService();
+ nodeHealthChecker = new NodeHealthCheckerService(
+ NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
+ nodeHealthChecker.init(conf);
+ }
@Test
public void testApplicationRecovery() throws Exception {
- YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
- conf.set(YarnConfiguration.NM_ADDRESS, "localhost:1234");
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "yarn_admin_user");
NMStateStoreService stateStore = new NMMemoryStateStoreService();
@@ -234,6 +290,91 @@ public class TestContainerManagerRecovery {
}
@Test
+ public void testContainerResizeRecovery() throws Exception {
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
+ NMStateStoreService stateStore = new NMMemoryStateStoreService();
+ stateStore.init(conf);
+ stateStore.start();
+ Context context = createContext(conf, stateStore);
+ ContainerManagerImpl cm = createContainerManager(context, delSrvc);
+ cm.init(conf);
+ cm.start();
+ // add an application by starting a container
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId attemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cid = ContainerId.newContainerId(attemptId, 1);
+ Map<String, String> containerEnv = Collections.emptyMap();
+ Map<String, ByteBuffer> serviceData = Collections.emptyMap();
+ Credentials containerCreds = new Credentials();
+ DataOutputBuffer dob = new DataOutputBuffer();
+ containerCreds.writeTokenStorageToStream(dob);
+ ByteBuffer containerTokens = ByteBuffer.wrap(dob.getData(), 0,
+ dob.getLength());
+ Map<ApplicationAccessType, String> acls = Collections.emptyMap();
+ File tmpDir = new File("target",
+ this.getClass().getSimpleName() + "-tmpDir");
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
+ PrintWriter fileWriter = new PrintWriter(scriptFile);
+ if (Shell.WINDOWS) {
+ fileWriter.println("@ping -n 100 127.0.0.1 >nul");
+ } else {
+ fileWriter.write("\numask 0");
+ fileWriter.write("\nexec sleep 100");
+ }
+ fileWriter.close();
+ FileContext localFS = FileContext.getLocalFSFileContext();
+ URL resource_alpha =
+ ConverterUtils.getYarnUrlFromPath(localFS
+ .makeQualified(new Path(scriptFile.getAbsolutePath())));
+ LocalResource rsrc_alpha = RecordFactoryProvider
+ .getRecordFactory(null).newRecordInstance(LocalResource.class);
+ rsrc_alpha.setResource(resource_alpha);
+ rsrc_alpha.setSize(-1);
+ rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+ rsrc_alpha.setType(LocalResourceType.FILE);
+ rsrc_alpha.setTimestamp(scriptFile.lastModified());
+ String destinationFile = "dest_file";
+ Map<String, LocalResource> localResources = new HashMap<>();
+ localResources.put(destinationFile, rsrc_alpha);
+ List<String> commands =
+ Arrays.asList(Shell.getRunScriptCommand(scriptFile));
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ localResources, containerEnv, commands, serviceData,
+ containerTokens, acls);
+ StartContainersResponse startResponse = startContainer(
+ context, cm, cid, clc, null);
+ assertTrue(startResponse.getFailedRequests().isEmpty());
+ assertEquals(1, context.getApplications().size());
+ Application app = context.getApplications().get(appId);
+ assertNotNull(app);
+ // make sure the container reaches RUNNING state
+ waitForNMContainerState(cm, cid,
+ org.apache.hadoop.yarn.server.nodemanager
+ .containermanager.container.ContainerState.RUNNING);
+ Resource targetResource = Resource.newInstance(2048, 2);
+ IncreaseContainersResourceResponse increaseResponse =
+ increaseContainersResource(context, cm, cid, targetResource);
+ assertTrue(increaseResponse.getFailedRequests().isEmpty());
+ // check status
+ ContainerStatus containerStatus = getContainerStatus(context, cm, cid);
+ assertEquals(targetResource, containerStatus.getCapability());
+ // restart and verify container is running and recovered
+ // to the correct size
+ cm.stop();
+ context = createContext(conf, stateStore);
+ cm = createContainerManager(context);
+ cm.init(conf);
+ cm.start();
+ assertEquals(1, context.getApplications().size());
+ app = context.getApplications().get(appId);
+ assertNotNull(app);
+ containerStatus = getContainerStatus(context, cm, cid);
+ assertEquals(targetResource, containerStatus.getCapability());
+ }
+
+ @Test
public void testContainerCleanupOnShutdown() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId attemptId =
@@ -257,10 +398,8 @@ public class TestContainerManagerRecovery {
LogAggregationContext.newInstance("includePattern", "excludePattern");
// verify containers are stopped on shutdown without recovery
- YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, false);
conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, false);
- conf.set(YarnConfiguration.NM_ADDRESS, "localhost:1234");
Context context = createContext(conf, new NMNullStateStoreService());
ContainerManagerImpl cm = spy(createContainerManager(context));
cm.init(conf);
@@ -306,12 +445,36 @@ public class TestContainerManagerRecovery {
verify(cm, never()).handle(isA(CMgrCompletedAppsEvent.class));
}
- private NMContext createContext(YarnConfiguration conf,
+ private ContainerManagerImpl createContainerManager(Context context,
+ DeletionService delSrvc) {
+ return new ContainerManagerImpl(context, exec, delSrvc,
+ mock(NodeStatusUpdater.class), metrics, dirsHandler) {
+ @Override
+ public void
+ setBlockNewContainerRequests(boolean blockNewContainerRequests) {
+ // do nothing
+ }
+ @Override
+ protected void authorizeGetAndStopContainerRequest(
+ ContainerId containerId, Container container,
+ boolean stopRequest, NMTokenIdentifier identifier)
+ throws YarnException {
+ if(container == null || container.getUser().equals("Fail")){
+ throw new YarnException("Reject this container");
+ }
+ }
+ };
+ }
+
+ private NMContext createContext(Configuration conf,
NMStateStoreService stateStore) {
NMContext context = new NMContext(new NMContainerTokenSecretManager(
conf), new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), stateStore);
-
+ new ApplicationACLsManager(conf), stateStore){
+ public int getHttpPort() {
+ return HTTP_PORT;
+ }
+ };
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.setKeyId(123);
@@ -349,6 +512,58 @@ public class TestContainerManagerRecovery {
});
}
+ private IncreaseContainersResourceResponse increaseContainersResource(
+ Context context, final ContainerManagerImpl cm, ContainerId cid,
+ Resource capability) throws Exception {
+ UserGroupInformation user = UserGroupInformation.createRemoteUser(
+ cid.getApplicationAttemptId().toString());
+ // construct container resource increase request
+ final List<Token> increaseTokens = new ArrayList<Token>();
+ // add increase request
+ Token containerToken = TestContainerManager.createContainerToken(
+ cid, 0, context.getNodeId(), user.getShortUserName(),
+ capability, context.getContainerTokenSecretManager(), null);
+ increaseTokens.add(containerToken);
+ final IncreaseContainersResourceRequest increaseRequest =
+ IncreaseContainersResourceRequest.newInstance(increaseTokens);
+ NMTokenIdentifier nmToken = new NMTokenIdentifier(
+ cid.getApplicationAttemptId(), context.getNodeId(),
+ user.getShortUserName(),
+ context.getNMTokenSecretManager().getCurrentKey().getKeyId());
+ user.addTokenIdentifier(nmToken);
+ return user.doAs(
+ new PrivilegedExceptionAction<IncreaseContainersResourceResponse>() {
+ @Override
+ public IncreaseContainersResourceResponse run() throws Exception {
+ return cm.increaseContainersResource(increaseRequest);
+ }
+ });
+ }
+
+ private ContainerStatus getContainerStatus(
+ Context context, final ContainerManagerImpl cm, ContainerId cid)
+ throws Exception {
+ UserGroupInformation user = UserGroupInformation.createRemoteUser(
+ cid.getApplicationAttemptId().toString());
+ NMTokenIdentifier nmToken = new NMTokenIdentifier(
+ cid.getApplicationAttemptId(), context.getNodeId(),
+ user.getShortUserName(),
+ context.getNMTokenSecretManager().getCurrentKey().getKeyId());
+ user.addTokenIdentifier(nmToken);
+ List<ContainerId> containerIds = new ArrayList<>();
+ containerIds.add(cid);
+ final GetContainerStatusesRequest gcsRequest =
+ GetContainerStatusesRequest.newInstance(containerIds);
+ return user.doAs(
+ new PrivilegedExceptionAction<ContainerStatus>() {
+ @Override
+ public ContainerStatus run() throws Exception {
+ return cm.getContainerStatuses(gcsRequest)
+ .getContainerStatuses().get(0);
+ }
+ });
+ }
+
private void waitForAppState(Application app, ApplicationState state)
throws Exception {
final int msecPerSleep = 10;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
index e0487e7..a1c95ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
@@ -122,9 +123,10 @@ public class NMMemoryStateStoreService extends NMStateStoreService {
rcsCopy.killed = rcs.killed;
rcsCopy.diagnostics = rcs.diagnostics;
rcsCopy.startRequest = rcs.startRequest;
+ rcsCopy.capability = rcs.capability;
result.add(rcsCopy);
}
- return new ArrayList<RecoveredContainerState>();
+ return result;
}
@Override
@@ -153,6 +155,13 @@ public class NMMemoryStateStoreService extends NMStateStoreService {
}
@Override
+ public synchronized void storeContainerResourceChanged(
+ ContainerId containerId, Resource capability) throws IOException {
+ RecoveredContainerState rcs = getRecoveredContainerState(containerId);
+ rcs.capability = capability;
+ }
+
+ @Override
public synchronized void storeContainerKilled(ContainerId containerId)
throws IOException {
RecoveredContainerState rcs = getRecoveredContainerState(containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d076b7ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 1804424..08b49e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -298,6 +298,17 @@ public class TestNMLeveldbStateStoreService {
assertEquals(containerReq, rcs.getStartRequest());
assertEquals(diags.toString(), rcs.getDiagnostics());
+ // increase the container size, and verify recovered
+ stateStore.storeContainerResourceChanged(containerId, Resource.newInstance(2468, 4));
+ restartStateStore();
+ recoveredContainers = stateStore.loadContainersState();
+ assertEquals(1, recoveredContainers.size());
+ rcs = recoveredContainers.get(0);
+ assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus());
+ assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
+ assertEquals(false, rcs.getKilled());
+ assertEquals(Resource.newInstance(2468, 4), rcs.getCapability());
+
// mark the container killed, add some more diags, and verify recovered
diags.append("some more diags for container");
stateStore.storeContainerDiagnostics(containerId, diags);
[15/32] hadoop git commit: HADOOP-12428. Fix inconsistency between
log-level guards and statements. Contributed by Jagadesh Kiran N and Jackie
Chang.
Posted by wa...@apache.org.
HADOOP-12428. Fix inconsistency between log-level guards and statements. Contributed by Jagadesh Kiran N and Jackie Chang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dfd807af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dfd807af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dfd807af
Branch: refs/heads/YARN-1197
Commit: dfd807afab0fae3839c9cc5d552aa0304444f956
Parents: 06022b8
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Sep 22 12:54:29 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Sep 22 12:54:29 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java | 4 ++--
.../java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java | 2 +-
.../apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java | 2 +-
.../org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java | 2 +-
.../apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java | 2 +-
.../org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java | 3 ++-
.../java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java | 2 +-
.../org/apache/hadoop/yarn/security/ContainerTokenSelector.java | 2 +-
.../java/org/apache/hadoop/yarn/security/NMTokenSelector.java | 2 +-
.../server/resourcemanager/scheduler/capacity/LeafQueue.java | 4 ++--
16 files changed, 27 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0c8550d..6bc3379 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -788,6 +788,9 @@ Release 2.8.0 - UNRELEASED
JarFile with other users when loading resource from URL in Configuration
class. (zxu)
+ HADOOP-12428. Fix inconsistency between log-level guards and statements.
+ (Jagadesh Kiran N and Jackie Chang via ozawa)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 944986c..44774a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -941,6 +941,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9111. Move hdfs-client protobuf convert methods from PBHelper to
PBHelperClient. (Mingliang Liu via wheat9)
+ HADOOP-12428. Fix inconsistency between log-level guards and statements.
+ (Jagadesh Kiran N and Jackie Chang via ozawa)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 4f6dc96..8c4e38a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -752,7 +752,7 @@ class DataXceiver extends Receiver implements Runnable {
mirrorInStatus = connectAck.getStatus();
firstBadLink = connectAck.getFirstBadLink();
if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) {
- LOG.info("Datanode " + targets.length +
+ LOG.debug("Datanode " + targets.length +
" got response for connect ack " +
" from downstream datanode with firstbadlink as " +
firstBadLink);
@@ -791,7 +791,7 @@ class DataXceiver extends Receiver implements Runnable {
// send connect-ack to source for clients and not transfer-RBW/Finalized
if (isClient && !isTransfer) {
if (LOG.isDebugEnabled() || mirrorInStatus != SUCCESS) {
- LOG.info("Datanode " + targets.length +
+ LOG.debug("Datanode " + targets.length +
" forwarding connect ack to upstream firstbadlink is " +
firstBadLink);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
index c6ae0d5..450ddee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
@@ -200,7 +200,7 @@ public class BackupImage extends FSImage {
assert backupInputStream.length() == 0 : "backup input stream is not empty";
try {
if (LOG.isTraceEnabled()) {
- LOG.debug("data:" + StringUtils.byteToHexString(data));
+ LOG.trace("data:" + StringUtils.byteToHexString(data));
}
FSEditLogLoader logLoader =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index a84dcf1..deb76b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -375,7 +375,7 @@ public class BootstrapStandby implements Tool, Configurable {
"or call saveNamespace on the active node.\n" +
"Error: " + e.getLocalizedMessage();
if (LOG.isDebugEnabled()) {
- LOG.fatal(msg, e);
+ LOG.debug(msg, e);
} else {
LOG.fatal(msg);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index cfca77c..59f90a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -260,7 +260,7 @@ public class EditLogTailer {
throw elie;
} finally {
if (editsLoaded > 0 || LOG.isDebugEnabled()) {
- LOG.info(String.format("Loaded %d edits starting from txid %d ",
+ LOG.debug(String.format("Loaded %d edits starting from txid %d ",
editsLoaded, lastTxnId));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 42ea011..bcdac1f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -387,6 +387,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6483. Replace deprecated method NameNode.getUri() with
DFSUtilClient.getNNUri() in TestMRCredentials. (Mingliang Liu via wheat9)
+ HADOOP-12428. Fix inconsistency between log-level guards and statements.
+ (Jagadesh Kiran N and Jackie Chang via ozawa)
+
OPTIMIZATIONS
MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 78b0dc4..74a2753 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -1122,7 +1122,7 @@ public class RMContainerAllocator extends RMContainerRequestor
assignedRequests.add(allocated, assigned.attemptID);
if (LOG.isDebugEnabled()) {
- LOG.info("Assigned container (" + allocated + ") "
+ LOG.debug("Assigned container (" + allocated + ") "
+ " to task " + assigned.attemptID + " on node "
+ allocated.getNodeId().toString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index 155711f..d612126 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -506,7 +506,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
- LOG.info("AFTER decResourceRequest:" + " applicationId="
+ LOG.debug("AFTER decResourceRequest:" + " applicationId="
+ applicationId.getId() + " priority=" + priority.getPriority()
+ " resourceName=" + resourceName + " numContainers="
+ remoteRequest.getNumContainers() + " #asks=" + ask.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
index 8dc7a65..8f31234 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java
@@ -112,7 +112,8 @@ public class UniformSizeInputFormat
FileSplit split = new FileSplit(listingFilePath, lastSplitStart,
lastPosition - lastSplitStart, null);
if (LOG.isDebugEnabled()) {
- LOG.info ("Creating split : " + split + ", bytes in split: " + currentSplitSize);
+ LOG.debug("Creating split : " + split + ", bytes in split: "
+ + currentSplitSize);
}
splits.add(split);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
index af554ff..471d140 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
@@ -130,7 +130,7 @@ public class SerialJobFactory extends JobFactory<JobStats> {
return;
}
if (LOG.isDebugEnabled()) {
- LOG.info(" job " + job.getName() + " completed ");
+ LOG.debug(" job " + job.getName() + " completed ");
}
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0e20d9a..329e743 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -457,6 +457,9 @@ Release 2.8.0 - UNRELEASED
YARN-3920. FairScheduler container reservation on a node should be
configurable to limit it to large containers (adhoot via asuresh)
+ HADOOP-12428. Fix inconsistency between log-level guards and statements.
+ (Jagadesh Kiran N and Jackie Chang via ozawa)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index b1324c1..4cf9aa0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -748,7 +748,7 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
if (LOG.isDebugEnabled()) {
- LOG.info("AFTER decResourceRequest:" + " applicationId="
+ LOG.debug("AFTER decResourceRequest:" + " applicationId="
+ " priority=" + priority.getPriority()
+ " resourceName=" + resourceName + " numContainers="
+ resourceRequestInfo.remoteRequest.getNumContainers()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java
index 5a3b494..a1c0a91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenSelector.java
@@ -46,7 +46,7 @@ public class ContainerTokenSelector implements
}
for (Token<? extends TokenIdentifier> token : tokens) {
if (LOG.isDebugEnabled()) {
- LOG.info("Looking for service: " + service + ". Current token is "
+ LOG.debug("Looking for service: " + service + ". Current token is "
+ token);
}
if (ContainerTokenIdentifier.KIND.equals(token.getKind()) &&
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java
index c57e4a2..43899b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/NMTokenSelector.java
@@ -42,7 +42,7 @@ public class NMTokenSelector implements
}
for (Token<? extends TokenIdentifier> token : tokens) {
if (LOG.isDebugEnabled()) {
- LOG.info("Looking for service: " + service + ". Current token is "
+ LOG.debug("Looking for service: " + service + ". Current token is "
+ token);
}
if (NMTokenIdentifier.KIND.equals(token.getKind()) &&
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd807af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index b43f658..869b49a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1163,8 +1163,8 @@ public class LeafQueue extends AbstractCSQueue {
metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
if (LOG.isDebugEnabled()) {
- LOG.info(getQueueName() +
- " user=" + userName +
+ LOG.debug(getQueueName() +
+ " user=" + userName +
" used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
" headroom = " + application.getHeadroom() +
" user-resources=" + user.getUsed()
[02/32] hadoop git commit: MAPREDUCE-6483. Replace deprecated method
NameNode.getUri() with DFSUtilClient.getNNUri() in TestMRCredentials.
Contributed by Mingliang Liu.
Posted by wa...@apache.org.
MAPREDUCE-6483. Replace deprecated method NameNode.getUri() with DFSUtilClient.getNNUri() in TestMRCredentials. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/602335df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/602335df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/602335df
Branch: refs/heads/YARN-1197
Commit: 602335dfe61a08be02ffa5a9c5b6c3f148926a7d
Parents: f97dc03
Author: Haohui Mai <wh...@apache.org>
Authored: Fri Sep 18 13:36:09 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Sep 18 13:36:09 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/mapreduce/security/TestMRCredentials.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/602335df/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a80a8f6..c67e494 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -384,6 +384,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6477. Replace usage of deprecated NameNode.DEFAULT_PORT in
TestFileSystem. (Mingliang Liu via wheat9)
+ MAPREDUCE-6483. Replace deprecated method NameNode.getUri() with
+ DFSUtilClient.getNNUri() in TestMRCredentials. (Mingliang Liu via wheat9)
+
OPTIMIZATIONS
MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/602335df/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
index 72639e2..85d60f0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
@@ -29,6 +29,7 @@ import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.mapred.JobConf;
@@ -111,9 +112,8 @@ public class TestMRCredentials {
Configuration jobConf = new JobConf(mrCluster.getConfig());
// provide namenodes names for the job to get the delegation tokens for
- //String nnUri = dfsCluster.getNameNode().getUri(namenode).toString();
NameNode nn = dfsCluster.getNameNode();
- URI nnUri = NameNode.getUri(nn.getNameNodeAddress());
+ URI nnUri = DFSUtilClient.getNNUri(nn.getNameNodeAddress());
jobConf.set(JobContext.JOB_NAMENODES, nnUri + "," + nnUri.toString());
[06/32] hadoop git commit: HADOOP-12360. Create StatsD metrics2 sink.
(Dave Marion via stevel)
Posted by wa...@apache.org.
HADOOP-12360. Create StatsD metrics2 sink. (Dave Marion via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c39ddc30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c39ddc30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c39ddc30
Branch: refs/heads/YARN-1197
Commit: c39ddc306d7efd93332fc1062ff4c9b59e94f3e1
Parents: e3ace31
Author: Steve Loughran <st...@apache.org>
Authored: Sat Sep 19 12:41:29 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Sep 19 12:41:42 2015 +0100
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../apache/hadoop/metrics2/package-info.java | 4 +-
.../apache/hadoop/metrics2/sink/StatsDSink.java | 218 +++++++++++++++++++
.../hadoop/metrics2/impl/TestStatsDMetrics.java | 122 +++++++++++
4 files changed, 345 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ddc30/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2bf5c9d..0c8550d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -546,6 +546,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-5732. Add SFTP FileSystem. (Ramtin Boustani and Inigo Goiri via
cdouglas)
+ HADOOP-12360. Create StatsD metrics2 sink. (Dave Marion via stevel)
+
IMPROVEMENTS
HADOOP-12271. Hadoop Jar Error Should Be More Explanatory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ddc30/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
index be21499..877c912 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
@@ -74,7 +74,9 @@
</dd>
<dt> <code>org.apache.hadoop.metrics2.sink</code></dt>
<dd>Builtin metrics sink implementations including the
- {@link org.apache.hadoop.metrics2.sink.FileSink}.
+ {@link org.apache.hadoop.metrics2.sink.FileSink},
+ {@link org.apache.hadoop.metrics2.sink.GraphiteSink}, and
+ {@link org.apache.hadoop.metrics2.sink.StatsDSink}.
</dd>
<dt> <code>org.apache.hadoop.metrics2.util</code></dt>
<dd>General utilities for implementing metrics sinks etc., including the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ddc30/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
new file mode 100644
index 0000000..b8059af
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricType;
+import org.apache.hadoop.metrics2.MetricsException;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.hadoop.metrics2.impl.MsInfo;
+import org.apache.hadoop.net.NetUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A metrics sink that writes metrics to a StatsD daemon.
+ * This sink will produce metrics of the form
+ * '[hostname].servicename.context.name.metricname:value|type'
+ * where hostname is optional. This is useful when sending to
+ * a daemon that is running on the localhost and will add the
+ * hostname to the metric (such as the
+ * <a href="https://collectd.org/">CollectD</a> StatsD plugin).
+ * <br/>
+ * To configure this plugin, you will need to add the following
+ * entries to your hadoop-metrics2.properties file:
+ * <br/>
+ * <pre>
+ * *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ * [prefix].sink.statsd.server.host=
+ * [prefix].sink.statsd.server.port=
+ * [prefix].sink.statsd.skip.hostname=true|false (optional)
+ * [prefix].sink.statsd.service.name=NameNode (name you want for service)
+ * </pre>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class StatsDSink implements MetricsSink, Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(StatsDSink.class);
+ private static final String PERIOD = ".";
+ private static final String SERVER_HOST_KEY = "server.host";
+ private static final String SERVER_PORT_KEY = "server.port";
+ private static final String HOST_NAME_KEY = "host.name";
+ private static final String SERVICE_NAME_KEY = "service.name";
+ private static final String SKIP_HOSTNAME_KEY = "skip.hostname";
+ private boolean skipHostname = false;
+ private String hostName = null;
+ private String serviceName = null;
+ private StatsD statsd = null;
+
+ @Override
+ public void init(SubsetConfiguration conf) {
+ // Get StatsD host configurations.
+ final String serverHost = conf.getString(SERVER_HOST_KEY);
+ final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY));
+
+ skipHostname = conf.getBoolean(SKIP_HOSTNAME_KEY, false);
+ if (!skipHostname) {
+ hostName = conf.getString(HOST_NAME_KEY, null);
+ if (null == hostName) {
+ hostName = NetUtils.getHostname();
+ }
+ }
+
+ serviceName = conf.getString(SERVICE_NAME_KEY, null);
+
+ statsd = new StatsD(serverHost, serverPort);
+ }
+
+ @Override
+ public void putMetrics(MetricsRecord record) {
+
+ String hn = hostName;
+ String ctx = record.context();
+ String sn = serviceName;
+
+ for (MetricsTag tag : record.tags()) {
+ if (tag.info().name().equals(MsInfo.Hostname.name())
+ && tag.value() != null) {
+ hn = tag.value();
+ } else if (tag.info().name().equals(MsInfo.Context.name())
+ && tag.value() != null) {
+ ctx = tag.value();
+ } else if (tag.info().name().equals(MsInfo.ProcessName.name())
+ && tag.value() != null) {
+ sn = tag.value();
+ }
+ }
+
+ StringBuilder buf = new StringBuilder();
+ if (!skipHostname && hn != null) {
+ int idx = hn.indexOf(".");
+ if (idx == -1) {
+ buf.append(hn).append(PERIOD);
+ } else {
+ buf.append(hn.substring(0, idx)).append(PERIOD);
+ }
+ }
+ buf.append(sn).append(PERIOD);
+ buf.append(ctx).append(PERIOD);
+ buf.append(record.name().replaceAll("\\.", "-")).append(PERIOD);
+
+ // Collect datapoints.
+ for (AbstractMetric metric : record.metrics()) {
+ String type = null;
+ if (metric.type().equals(MetricType.COUNTER)) {
+ type = "c";
+ } else if (metric.type().equals(MetricType.GAUGE)) {
+ type = "g";
+ }
+ StringBuilder line = new StringBuilder();
+ line.append(buf.toString())
+ .append(metric.name().replace(' ', '_'))
+ .append(":")
+ .append(metric.value())
+ .append("|")
+ .append(type);
+ writeMetric(line.toString());
+ }
+
+ }
+
+ public void writeMetric(String line) {
+ try {
+ statsd.write(line);
+ } catch (IOException e) {
+ LOG.warn("Error sending metrics to StatsD", e);
+ throw new MetricsException("Error writing metric to StatsD", e);
+ }
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public void close() throws IOException {
+ statsd.close();
+ }
+
+ /**
+ * Class that sends UDP packets to StatsD daemon.
+ *
+ */
+ public static class StatsD {
+
+ private DatagramSocket socket = null;
+ private DatagramPacket packet = null;
+ private String serverHost;
+ private int serverPort;
+
+ public StatsD(String serverHost, int serverPort) {
+ this.serverHost = serverHost;
+ this.serverPort = serverPort;
+ }
+
+ public void createSocket() throws IOException {
+ try {
+ InetSocketAddress address =
+ new InetSocketAddress(this.serverHost, this.serverPort);
+ socket = new DatagramSocket();
+ packet =
+ new DatagramPacket("".getBytes(StandardCharsets.UTF_8), 0, 0,
+ address.getAddress(), this.serverPort);
+ } catch (IOException ioe) {
+ throw NetUtils.wrapException(this.serverHost, this.serverPort,
+ "localhost", 0, ioe);
+ }
+ }
+
+ public void write(String msg) throws IOException {
+ if (null == socket) {
+ createSocket();
+ }
+ LOG.debug("Sending metric: {}", msg);
+ packet.setData(msg.getBytes(StandardCharsets.UTF_8));
+ socket.send(packet);
+ }
+
+ public void close() throws IOException {
+ try {
+ if (socket != null) {
+ socket.close();
+ }
+ } finally {
+ socket = null;
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c39ddc30/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java
new file mode 100644
index 0000000..355c1cb
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.impl;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricType;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.hadoop.metrics2.sink.StatsDSink;
+import org.apache.hadoop.metrics2.sink.StatsDSink.StatsD;
+import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
+
+public class TestStatsDMetrics {
+
+ private AbstractMetric makeMetric(String name, Number value,
+ MetricType type) {
+ AbstractMetric metric = mock(AbstractMetric.class);
+ when(metric.name()).thenReturn(name);
+ when(metric.value()).thenReturn(value);
+ when(metric.type()).thenReturn(type);
+ return metric;
+ }
+
+ @Test(timeout=3000)
+ public void testPutMetrics() throws IOException, InterruptedException {
+ final StatsDSink sink = new StatsDSink();
+ List<MetricsTag> tags = new ArrayList<MetricsTag>();
+ tags.add(new MetricsTag(MsInfo.Hostname, "host"));
+ tags.add(new MetricsTag(MsInfo.Context, "jvm"));
+ tags.add(new MetricsTag(MsInfo.ProcessName, "process"));
+ Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
+ metrics.add(makeMetric("foo1", 1.25, MetricType.COUNTER));
+ metrics.add(makeMetric("foo2", 2.25, MetricType.GAUGE));
+ final MetricsRecord record =
+ new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
+
+ try (DatagramSocket sock = new DatagramSocket()) {
+ sock.setReceiveBufferSize(8192);
+ final StatsDSink.StatsD mockStatsD =
+ new StatsD(sock.getLocalAddress().getHostName(),
+ sock.getLocalPort());
+ Whitebox.setInternalState(sink, "statsd", mockStatsD);
+ final DatagramPacket p = new DatagramPacket(new byte[8192], 8192);
+ sink.putMetrics(record);
+ sock.receive(p);
+
+ String result =new String(p.getData(), 0, p.getLength(),
+ Charset.forName("UTF-8"));
+ assertTrue(
+ "Received data did not match data sent",
+ result.equals("host.process.jvm.Context.foo1:1.25|c") ||
+ result.equals("host.process.jvm.Context.foo2:2.25|g"));
+
+ } finally {
+ sink.close();
+ }
+ }
+
+ @Test(timeout=3000)
+ public void testPutMetrics2() throws IOException {
+ StatsDSink sink = new StatsDSink();
+ List<MetricsTag> tags = new ArrayList<MetricsTag>();
+ tags.add(new MetricsTag(MsInfo.Hostname, null));
+ tags.add(new MetricsTag(MsInfo.Context, "jvm"));
+ tags.add(new MetricsTag(MsInfo.ProcessName, "process"));
+ Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
+ metrics.add(makeMetric("foo1", 1, MetricType.COUNTER));
+ metrics.add(makeMetric("foo2", 2, MetricType.GAUGE));
+ MetricsRecord record =
+ new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
+
+ try (DatagramSocket sock = new DatagramSocket()) {
+ sock.setReceiveBufferSize(8192);
+ final StatsDSink.StatsD mockStatsD =
+ new StatsD(sock.getLocalAddress().getHostName(),
+ sock.getLocalPort());
+ Whitebox.setInternalState(sink, "statsd", mockStatsD);
+ final DatagramPacket p = new DatagramPacket(new byte[8192], 8192);
+ sink.putMetrics(record);
+ sock.receive(p);
+ String result =
+ new String(p.getData(), 0, p.getLength(), Charset.forName("UTF-8"));
+
+ assertTrue("Received data did not match data sent",
+ result.equals("process.jvm.Context.foo1:1|c") ||
+ result.equals("process.jvm.Context.foo2:2|g"));
+ } finally {
+ sink.close();
+ }
+ }
+
+}
[29/32] hadoop git commit: YARN-1651. CapacityScheduler side changes
to support container resize. Contributed by Wangda Tan
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 869b49a..2ab060e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -60,10 +59,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.Lock;
@@ -730,17 +729,22 @@ public class LeafQueue extends AbstractCSQueue {
}
private void handleExcessReservedContainer(Resource clusterResource,
- CSAssignment assignment) {
+ CSAssignment assignment, FiCaSchedulerNode node, FiCaSchedulerApp app) {
if (assignment.getExcessReservation() != null) {
RMContainer excessReservedContainer = assignment.getExcessReservation();
-
- completedContainer(clusterResource, assignment.getApplication(),
- scheduler.getNode(excessReservedContainer.getAllocatedNode()),
- excessReservedContainer,
- SchedulerUtils.createAbnormalContainerStatus(
- excessReservedContainer.getContainerId(),
- SchedulerUtils.UNRESERVED_CONTAINER),
- RMContainerEventType.RELEASED, null, false);
+
+ if (excessReservedContainer.hasIncreaseReservation()) {
+ unreserveIncreasedContainer(clusterResource,
+ app, node, excessReservedContainer);
+ } else {
+ completedContainer(clusterResource, assignment.getApplication(),
+ scheduler.getNode(excessReservedContainer.getAllocatedNode()),
+ excessReservedContainer,
+ SchedulerUtils.createAbnormalContainerStatus(
+ excessReservedContainer.getContainerId(),
+ SchedulerUtils.UNRESERVED_CONTAINER),
+ RMContainerEventType.RELEASED, null, false);
+ }
assignment.setExcessReservation(null);
}
@@ -766,7 +770,8 @@ public class LeafQueue extends AbstractCSQueue {
CSAssignment assignment =
application.assignContainers(clusterResource, node,
currentResourceLimits, schedulingMode, reservedContainer);
- handleExcessReservedContainer(clusterResource, assignment);
+ handleExcessReservedContainer(clusterResource, assignment, node,
+ application);
return assignment;
}
}
@@ -824,7 +829,8 @@ public class LeafQueue extends AbstractCSQueue {
// Did we schedule or reserve a container?
Resource assigned = assignment.getResource();
- handleExcessReservedContainer(clusterResource, assignment);
+ handleExcessReservedContainer(clusterResource, assignment, node,
+ application);
if (Resources.greaterThan(resourceCalculator, clusterResource, assigned,
Resources.none())) {
@@ -836,7 +842,8 @@ public class LeafQueue extends AbstractCSQueue {
// Book-keeping
// Note: Update headroom to account for current allocation too...
allocateResource(clusterResource, application, assigned,
- node.getPartition(), reservedOrAllocatedRMContainer);
+ node.getPartition(), reservedOrAllocatedRMContainer,
+ assignment.isIncreasedAllocation());
// Done
return assignment;
@@ -1086,6 +1093,37 @@ public class LeafQueue extends AbstractCSQueue {
}
return true;
}
+
+ @Override
+ public void unreserveIncreasedContainer(Resource clusterResource,
+ FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer) {
+ boolean removed = false;
+ Priority priority = null;
+
+ synchronized (this) {
+ if (rmContainer.getContainer() != null) {
+ priority = rmContainer.getContainer().getPriority();
+ }
+
+ if (null != priority) {
+ removed = app.unreserve(rmContainer.getContainer().getPriority(), node,
+ rmContainer);
+ }
+
+ if (removed) {
+ // Inform the ordering policy
+ orderingPolicy.containerReleased(app, rmContainer);
+
+ releaseResource(clusterResource, app, rmContainer.getReservedResource(),
+ node.getPartition(), rmContainer, true);
+ }
+ }
+
+ if (removed) {
+ getParent().unreserveIncreasedContainer(clusterResource, app, node,
+ rmContainer);
+ }
+ }
@Override
public void completedContainer(Resource clusterResource,
@@ -1093,6 +1131,15 @@ public class LeafQueue extends AbstractCSQueue {
ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
boolean sortQueues) {
if (application != null) {
+ // unreserve container increase request if it previously reserved.
+ if (rmContainer.hasIncreaseReservation()) {
+ unreserveIncreasedContainer(clusterResource, application, node,
+ rmContainer);
+ }
+
+ // Remove container increase request if it exists
+ application.removeIncreaseRequest(node.getNodeID(),
+ rmContainer.getAllocatedPriority(), rmContainer.getContainerId());
boolean removed = false;
@@ -1123,7 +1170,7 @@ public class LeafQueue extends AbstractCSQueue {
orderingPolicy.containerReleased(application, rmContainer);
releaseResource(clusterResource, application, container.getResource(),
- node.getPartition(), rmContainer);
+ node.getPartition(), rmContainer, false);
}
}
@@ -1137,8 +1184,10 @@ public class LeafQueue extends AbstractCSQueue {
synchronized void allocateResource(Resource clusterResource,
SchedulerApplicationAttempt application, Resource resource,
- String nodePartition, RMContainer rmContainer) {
- super.allocateResource(clusterResource, resource, nodePartition);
+ String nodePartition, RMContainer rmContainer,
+ boolean isIncreasedAllocation) {
+ super.allocateResource(clusterResource, resource, nodePartition,
+ isIncreasedAllocation);
// handle ignore exclusivity container
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
@@ -1174,8 +1223,9 @@ public class LeafQueue extends AbstractCSQueue {
synchronized void releaseResource(Resource clusterResource,
FiCaSchedulerApp application, Resource resource, String nodePartition,
- RMContainer rmContainer) {
- super.releaseResource(clusterResource, resource, nodePartition);
+ RMContainer rmContainer, boolean isChangeResource) {
+ super.releaseResource(clusterResource, resource, nodePartition,
+ isChangeResource);
// handle ignore exclusivity container
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
@@ -1363,7 +1413,7 @@ public class LeafQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, attempt, rmContainer.getContainer()
- .getResource(), node.getPartition(), rmContainer);
+ .getResource(), node.getPartition(), rmContainer, false);
}
getParent().recoverContainer(clusterResource, attempt, rmContainer);
}
@@ -1412,7 +1462,7 @@ public class LeafQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), node.getPartition(), rmContainer);
+ .getResource(), node.getPartition(), rmContainer, false);
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
@@ -1430,7 +1480,7 @@ public class LeafQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
releaseResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), node.getPartition(), rmContainer);
+ .getResource(), node.getPartition(), rmContainer, false);
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
@@ -1482,6 +1532,39 @@ public class LeafQueue extends AbstractCSQueue {
public Priority getDefaultApplicationPriority() {
return defaultAppPriorityPerQueue;
}
+
+ @Override
+ public void decreaseContainer(Resource clusterResource,
+ SchedContainerChangeRequest decreaseRequest,
+ FiCaSchedulerApp app) {
+ // If the container being decreased is reserved, we need to unreserve it
+ // first.
+ RMContainer rmContainer = decreaseRequest.getRMContainer();
+ if (rmContainer.hasIncreaseReservation()) {
+ unreserveIncreasedContainer(clusterResource, app,
+ (FiCaSchedulerNode)decreaseRequest.getSchedulerNode(), rmContainer);
+ }
+
+ // Delta capacity is negative when it's a decrease request
+ Resource absDelta = Resources.negate(decreaseRequest.getDeltaCapacity());
+
+ synchronized (this) {
+ // Delta is negative when it's a decrease request
+ releaseResource(clusterResource, app, absDelta,
+ decreaseRequest.getNodePartition(), decreaseRequest.getRMContainer(),
+ true);
+ // Notify application
+ app.decreaseContainer(decreaseRequest);
+ // Notify node
+ decreaseRequest.getSchedulerNode()
+ .decreaseContainer(decreaseRequest.getContainerId(), absDelta);
+ }
+
+ // Notify parent
+ if (getParent() != null) {
+ getParent().decreaseContainer(clusterResource, decreaseRequest, app);
+ }
+ }
public synchronized OrderingPolicy<FiCaSchedulerApp>
getPendingAppsOrderingPolicy() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index e01204c..badab72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -430,7 +431,7 @@ public class ParentQueue extends AbstractCSQueue {
assignedToChild.getResource(), Resources.none())) {
// Track resource utilization for the parent-queue
super.allocateResource(clusterResource, assignedToChild.getResource(),
- node.getPartition());
+ node.getPartition(), assignedToChild.isIncreasedAllocation());
// Track resource utilization in this pass of the scheduler
Resources
@@ -454,6 +455,8 @@ public class ParentQueue extends AbstractCSQueue {
.addAll(
assignedToChild.getAssignmentInformation()
.getReservationDetails());
+ assignment.setIncreasedAllocation(assignedToChild
+ .isIncreasedAllocation());
LOG.info("assignedContainer" +
" queue=" + getQueueName() +
@@ -616,6 +619,73 @@ public class ParentQueue extends AbstractCSQueue {
}
}
+ private synchronized void internalReleaseResource(Resource clusterResource,
+ FiCaSchedulerNode node, Resource releasedResource, boolean changeResource,
+ CSQueue completedChildQueue, boolean sortQueues) {
+ super.releaseResource(clusterResource,
+ releasedResource, node.getPartition(),
+ changeResource);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("completedContainer " + this + ", cluster=" + clusterResource);
+ }
+
+ // Note that this is using an iterator on the childQueues so this can't
+ // be called if already within an iterator for the childQueues. Like
+ // from assignContainersToChildQueues.
+ if (sortQueues) {
+ // reinsert the updated queue
+ for (Iterator<CSQueue> iter = childQueues.iterator(); iter.hasNext();) {
+ CSQueue csqueue = iter.next();
+ if (csqueue.equals(completedChildQueue)) {
+ iter.remove();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Re-sorting completed queue: " + csqueue);
+ }
+ childQueues.add(csqueue);
+ break;
+ }
+ }
+ }
+
+ // If we skipped sort queue this time, we need to resort queues to make
+ // sure we allocate from least usage (or order defined by queue policy)
+ // queues.
+ needToResortQueuesAtNextAllocation = !sortQueues;
+ }
+
+ @Override
+ public void decreaseContainer(Resource clusterResource,
+ SchedContainerChangeRequest decreaseRequest, FiCaSchedulerApp app) {
+ // delta capacity is negative when it's a decrease request
+ Resource absDeltaCapacity =
+ Resources.negate(decreaseRequest.getDeltaCapacity());
+
+ internalReleaseResource(clusterResource,
+ csContext.getNode(decreaseRequest.getNodeId()), absDeltaCapacity, false,
+ null, false);
+
+ // Inform the parent
+ if (parent != null) {
+ parent.decreaseContainer(clusterResource, decreaseRequest, app);
+ }
+ }
+
+ @Override
+ public void unreserveIncreasedContainer(Resource clusterResource,
+ FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer) {
+ if (app != null) {
+ internalReleaseResource(clusterResource, node,
+ rmContainer.getReservedResource(), false, null, false);
+
+ // Inform the parent
+ if (parent != null) {
+ parent.unreserveIncreasedContainer(clusterResource, app, node,
+ rmContainer);
+ }
+ }
+ }
+
@Override
public void completedContainer(Resource clusterResource,
FiCaSchedulerApp application, FiCaSchedulerNode node,
@@ -623,40 +693,9 @@ public class ParentQueue extends AbstractCSQueue {
RMContainerEventType event, CSQueue completedChildQueue,
boolean sortQueues) {
if (application != null) {
- // Careful! Locking order is important!
- // Book keeping
- synchronized (this) {
- super.releaseResource(clusterResource, rmContainer.getContainer()
- .getResource(), node.getPartition());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("completedContainer " + this + ", cluster=" + clusterResource);
- }
-
- // Note that this is using an iterator on the childQueues so this can't
- // be called if already within an iterator for the childQueues. Like
- // from assignContainersToChildQueues.
- if (sortQueues) {
- // reinsert the updated queue
- for (Iterator<CSQueue> iter = childQueues.iterator();
- iter.hasNext();) {
- CSQueue csqueue = iter.next();
- if(csqueue.equals(completedChildQueue)) {
- iter.remove();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Re-sorting completed queue: " + csqueue);
- }
- childQueues.add(csqueue);
- break;
- }
- }
- }
-
- // If we skipped sort queue this time, we need to resort queues to make
- // sure we allocate from least usage (or order defined by queue policy)
- // queues.
- needToResortQueuesAtNextAllocation = !sortQueues;
- }
+ internalReleaseResource(clusterResource, node,
+ rmContainer.getContainer().getResource(), false, completedChildQueue,
+ sortQueues);
// Inform the parent
if (parent != null) {
@@ -698,7 +737,7 @@ public class ParentQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
super.allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), node.getPartition());
+ .getResource(), node.getPartition(), false);
}
if (parent != null) {
parent.recoverContainer(clusterResource, attempt, rmContainer);
@@ -726,7 +765,7 @@ public class ParentQueue extends AbstractCSQueue {
FiCaSchedulerNode node =
scheduler.getNode(rmContainer.getContainer().getNodeId());
super.allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), node.getPartition());
+ .getResource(), node.getPartition(), false);
LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
@@ -746,7 +785,7 @@ public class ParentQueue extends AbstractCSQueue {
scheduler.getNode(rmContainer.getContainer().getNodeId());
super.releaseResource(clusterResource,
rmContainer.getContainer().getResource(),
- node.getPartition());
+ node.getPartition(), false);
LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
new file mode 100644
index 0000000..b986b1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * For an application, resource limits and resource requests, decide how to
+ * allocate container. This is to make application resource allocation logic
+ * extensible.
+ */
+public abstract class AbstractContainerAllocator {
+ private static final Log LOG = LogFactory.getLog(AbstractContainerAllocator.class);
+
+ FiCaSchedulerApp application;
+ final ResourceCalculator rc;
+ final RMContext rmContext;
+
+ public AbstractContainerAllocator(FiCaSchedulerApp application,
+ ResourceCalculator rc, RMContext rmContext) {
+ this.application = application;
+ this.rc = rc;
+ this.rmContext = rmContext;
+ }
+
+ protected CSAssignment getCSAssignmentFromAllocateResult(
+ Resource clusterResource, ContainerAllocation result,
+ RMContainer rmContainer) {
+ // Handle skipped
+ boolean skipped =
+ (result.getAllocationState() == AllocationState.APP_SKIPPED);
+ CSAssignment assignment = new CSAssignment(skipped);
+ assignment.setApplication(application);
+
+ // Handle excess reservation
+ assignment.setExcessReservation(result.getContainerToBeUnreserved());
+
+ // If we allocated something
+ if (Resources.greaterThan(rc, clusterResource,
+ result.getResourceToBeAllocated(), Resources.none())) {
+ Resource allocatedResource = result.getResourceToBeAllocated();
+ Container updatedContainer = result.getUpdatedContainer();
+
+ assignment.setResource(allocatedResource);
+ assignment.setType(result.getContainerNodeType());
+
+ if (result.getAllocationState() == AllocationState.RESERVED) {
+ // This is a reserved container
+ LOG.info("Reserved container " + " application="
+ + application.getApplicationId() + " resource=" + allocatedResource
+ + " queue=" + this.toString() + " cluster=" + clusterResource);
+ assignment.getAssignmentInformation().addReservationDetails(
+ updatedContainer.getId(),
+ application.getCSLeafQueue().getQueuePath());
+ assignment.getAssignmentInformation().incrReservations();
+ Resources.addTo(assignment.getAssignmentInformation().getReserved(),
+ allocatedResource);
+ } else if (result.getAllocationState() == AllocationState.ALLOCATED){
+ // This is a new container
+ // Inform the ordering policy
+ LOG.info("assignedContainer" + " application attempt="
+ + application.getApplicationAttemptId() + " container="
+ + updatedContainer.getId() + " queue=" + this + " clusterResource="
+ + clusterResource);
+
+ application
+ .getCSLeafQueue()
+ .getOrderingPolicy()
+ .containerAllocated(application,
+ application.getRMContainer(updatedContainer.getId()));
+
+ assignment.getAssignmentInformation().addAllocationDetails(
+ updatedContainer.getId(),
+ application.getCSLeafQueue().getQueuePath());
+ assignment.getAssignmentInformation().incrAllocations();
+ Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
+ allocatedResource);
+
+ if (rmContainer != null) {
+ assignment.setFulfilledReservation(true);
+ }
+ }
+ }
+
+ return assignment;
+ }
+
+ /**
+ * allocate needs to handle following stuffs:
+ *
+ * <ul>
+ * <li>Select request: Select a request to allocate. E.g. select a resource
+ * request based on requirement/priority/locality.</li>
+ * <li>Check if a given resource can be allocated based on resource
+ * availability</li>
+ * <li>Do allocation: this will decide/create allocated/reserved
+ * container, this will also update metrics</li>
+ * </ul>
+ */
+ public abstract CSAssignment assignContainers(Resource clusterResource,
+ FiCaSchedulerNode node, SchedulingMode schedulingMode,
+ ResourceLimits resourceLimits, RMContainer reservedContainer);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
index 6e296cd..3be8e0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
@@ -18,13 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
@@ -33,118 +30,50 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-/**
- * For an application, resource limits and resource requests, decide how to
- * allocate container. This is to make application resource allocation logic
- * extensible.
- */
-public abstract class ContainerAllocator {
- private static final Log LOG = LogFactory.getLog(ContainerAllocator.class);
+public class ContainerAllocator extends AbstractContainerAllocator {
+ AbstractContainerAllocator increaseContainerAllocator;
+ AbstractContainerAllocator regularContainerAllocator;
- FiCaSchedulerApp application;
- final ResourceCalculator rc;
- final RMContext rmContext;
-
public ContainerAllocator(FiCaSchedulerApp application,
ResourceCalculator rc, RMContext rmContext) {
- this.application = application;
- this.rc = rc;
- this.rmContext = rmContext;
- }
+ super(application, rc, rmContext);
- protected boolean checkHeadroom(Resource clusterResource,
- ResourceLimits currentResourceLimits, Resource required,
- FiCaSchedulerNode node) {
- // If headroom + currentReservation < required, we cannot allocate this
- // require
- Resource resourceCouldBeUnReserved = application.getCurrentReservation();
- if (!application.getCSLeafQueue().getReservationContinueLooking()
- || !node.getPartition().equals(RMNodeLabelsManager.NO_LABEL)) {
- // If we don't allow reservation continuous looking, OR we're looking at
- // non-default node partition, we won't allow to unreserve before
- // allocation.
- resourceCouldBeUnReserved = Resources.none();
- }
- return Resources.greaterThanOrEqual(rc, clusterResource, Resources.add(
- currentResourceLimits.getHeadroom(), resourceCouldBeUnReserved),
- required);
+ increaseContainerAllocator =
+ new IncreaseContainerAllocator(application, rc, rmContext);
+ regularContainerAllocator =
+ new RegularContainerAllocator(application, rc, rmContext);
}
- protected CSAssignment getCSAssignmentFromAllocateResult(
- Resource clusterResource, ContainerAllocation result,
- RMContainer rmContainer) {
- // Handle skipped
- boolean skipped =
- (result.getAllocationState() == AllocationState.APP_SKIPPED);
- CSAssignment assignment = new CSAssignment(skipped);
- assignment.setApplication(application);
-
- // Handle excess reservation
- assignment.setExcessReservation(result.getContainerToBeUnreserved());
-
- // If we allocated something
- if (Resources.greaterThan(rc, clusterResource,
- result.getResourceToBeAllocated(), Resources.none())) {
- Resource allocatedResource = result.getResourceToBeAllocated();
- Container updatedContainer = result.getUpdatedContainer();
-
- assignment.setResource(allocatedResource);
- assignment.setType(result.getContainerNodeType());
-
- if (result.getAllocationState() == AllocationState.RESERVED) {
- // This is a reserved container
- LOG.info("Reserved container " + " application="
- + application.getApplicationId() + " resource=" + allocatedResource
- + " queue=" + this.toString() + " cluster=" + clusterResource);
- assignment.getAssignmentInformation().addReservationDetails(
- updatedContainer.getId(),
- application.getCSLeafQueue().getQueuePath());
- assignment.getAssignmentInformation().incrReservations();
- Resources.addTo(assignment.getAssignmentInformation().getReserved(),
- allocatedResource);
- } else if (result.getAllocationState() == AllocationState.ALLOCATED){
- // This is a new container
- // Inform the ordering policy
- LOG.info("assignedContainer" + " application attempt="
- + application.getApplicationAttemptId() + " container="
- + updatedContainer.getId() + " queue=" + this + " clusterResource="
- + clusterResource);
-
- application
- .getCSLeafQueue()
- .getOrderingPolicy()
- .containerAllocated(application,
- application.getRMContainer(updatedContainer.getId()));
-
- assignment.getAssignmentInformation().addAllocationDetails(
- updatedContainer.getId(),
- application.getCSLeafQueue().getQueuePath());
- assignment.getAssignmentInformation().incrAllocations();
- Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
- allocatedResource);
-
- if (rmContainer != null) {
- assignment.setFulfilledReservation(true);
- }
+ @Override
+ public CSAssignment assignContainers(Resource clusterResource,
+ FiCaSchedulerNode node, SchedulingMode schedulingMode,
+ ResourceLimits resourceLimits, RMContainer reservedContainer) {
+ if (reservedContainer != null) {
+ if (reservedContainer.getState() == RMContainerState.RESERVED) {
+ // It's a regular container
+ return regularContainerAllocator.assignContainers(clusterResource,
+ node, schedulingMode, resourceLimits, reservedContainer);
+ } else {
+ // It's a increase container
+ return increaseContainerAllocator.assignContainers(clusterResource,
+ node, schedulingMode, resourceLimits, reservedContainer);
+ }
+ } else {
+ /*
+ * Try to allocate increase container first, and if we failed to allocate
+ * anything, we will try to allocate regular container
+ */
+ CSAssignment assign =
+ increaseContainerAllocator.assignContainers(clusterResource, node,
+ schedulingMode, resourceLimits, null);
+ if (Resources.greaterThan(rc, clusterResource, assign.getResource(),
+ Resources.none())) {
+ return assign;
}
+
+ return regularContainerAllocator.assignContainers(clusterResource, node,
+ schedulingMode, resourceLimits, null);
}
-
- return assignment;
}
-
- /**
- * allocate needs to handle following stuffs:
- *
- * <ul>
- * <li>Select request: Select a request to allocate. E.g. select a resource
- * request based on requirement/priority/locality.</li>
- * <li>Check if a given resource can be allocated based on resource
- * availability</li>
- * <li>Do allocation: this will decide/create allocated/reserved
- * container, this will also update metrics</li>
- * </ul>
- */
- public abstract CSAssignment assignContainers(Resource clusterResource,
- FiCaSchedulerNode node, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits, RMContainer reservedContainer);
-}
\ No newline at end of file
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
new file mode 100644
index 0000000..9350adc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/IncreaseContainerAllocator.java
@@ -0,0 +1,365 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+public class IncreaseContainerAllocator extends AbstractContainerAllocator {
+ private static final Log LOG =
+ LogFactory.getLog(IncreaseContainerAllocator.class);
+
+ public IncreaseContainerAllocator(FiCaSchedulerApp application,
+ ResourceCalculator rc, RMContext rmContext) {
+ super(application, rc, rmContext);
+ }
+
+ /**
+ * Quick check if we can allocate anything here:
+ * We will not continue if:
+ * - Headroom doesn't support allocate minimumAllocation
+ * -
+ */
+ private boolean checkHeadroom(Resource clusterResource,
+ ResourceLimits currentResourceLimits, Resource required) {
+ return Resources.greaterThanOrEqual(rc, clusterResource,
+ currentResourceLimits.getHeadroom(), required);
+ }
+
+ private CSAssignment createReservedIncreasedCSAssignment(
+ SchedContainerChangeRequest request) {
+ CSAssignment assignment =
+ new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
+ application, false, false);
+ Resources.addTo(assignment.getAssignmentInformation().getReserved(),
+ request.getDeltaCapacity());
+ assignment.getAssignmentInformation().incrReservations();
+ assignment.getAssignmentInformation().addReservationDetails(
+ request.getContainerId(), application.getCSLeafQueue().getQueuePath());
+ assignment.setIncreasedAllocation(true);
+
+ LOG.info("Reserved increase container request:" + request.toString());
+
+ return assignment;
+ }
+
+ private CSAssignment createSuccessfullyIncreasedCSAssignment(
+ SchedContainerChangeRequest request, boolean fromReservation) {
+ CSAssignment assignment =
+ new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
+ application, false, fromReservation);
+ Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
+ request.getDeltaCapacity());
+ assignment.getAssignmentInformation().incrAllocations();
+ assignment.getAssignmentInformation().addAllocationDetails(
+ request.getContainerId(), application.getCSLeafQueue().getQueuePath());
+ assignment.setIncreasedAllocation(true);
+
+ // notify application
+ application
+ .getCSLeafQueue()
+ .getOrderingPolicy()
+ .containerAllocated(application,
+ application.getRMContainer(request.getContainerId()));
+
+ LOG.info("Approved increase container request:" + request.toString()
+ + " fromReservation=" + fromReservation);
+
+ return assignment;
+ }
+
+ private CSAssignment allocateIncreaseRequestFromReservedContainer(
+ SchedulerNode node, Resource cluster,
+ SchedContainerChangeRequest increaseRequest) {
+ if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
+ node.getAvailableResource())) {
+ // OK, we can allocate this increase request
+ // Unreserve it first
+ application.unreserve(increaseRequest.getPriority(),
+ (FiCaSchedulerNode) node, increaseRequest.getRMContainer());
+
+ // Notify application
+ application.increaseContainer(increaseRequest);
+
+ // Notify node
+ node.increaseContainer(increaseRequest.getContainerId(),
+ increaseRequest.getDeltaCapacity());
+
+ return createSuccessfullyIncreasedCSAssignment(increaseRequest, true);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to allocate reserved increase request:"
+ + increaseRequest.toString()
+ + ". There's no enough available resource");
+ }
+
+ // We still cannot allocate this container, will wait for next turn
+ return CSAssignment.SKIP_ASSIGNMENT;
+ }
+ }
+
+ private CSAssignment allocateIncreaseRequest(FiCaSchedulerNode node,
+ Resource cluster, SchedContainerChangeRequest increaseRequest) {
+ if (Resources.fitsIn(rc, cluster, increaseRequest.getDeltaCapacity(),
+ node.getAvailableResource())) {
+ // Notify node
+ node.increaseContainer(increaseRequest.getContainerId(),
+ increaseRequest.getDeltaCapacity());
+
+ // OK, we can allocate this increase request
+ // Notify application
+ application.increaseContainer(increaseRequest);
+ return createSuccessfullyIncreasedCSAssignment(increaseRequest, false);
+ } else {
+ boolean reservationSucceeded =
+ application.reserveIncreasedContainer(increaseRequest.getPriority(),
+ node, increaseRequest.getRMContainer(),
+ increaseRequest.getDeltaCapacity());
+
+ if (reservationSucceeded) {
+ // We cannot allocate this container, but since queue capacity /
+ // user-limit matches, we can reserve this container on this node.
+ return createReservedIncreasedCSAssignment(increaseRequest);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reserve increase request=" + increaseRequest.toString()
+ + " failed. Skipping..");
+ }
+ return CSAssignment.SKIP_ASSIGNMENT;
+ }
+ }
+ }
+
+ @Override
+ public CSAssignment assignContainers(Resource clusterResource,
+ FiCaSchedulerNode node, SchedulingMode schedulingMode,
+ ResourceLimits resourceLimits, RMContainer reservedContainer) {
+ AppSchedulingInfo sinfo = application.getAppSchedulingInfo();
+ NodeId nodeId = node.getNodeID();
+
+ if (reservedContainer == null) {
+ // Do we have increase request on this node?
+ if (!sinfo.hasIncreaseRequest(nodeId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip allocating increase request since we don't have any"
+ + " increase request on this node=" + node.getNodeID());
+ }
+
+ return CSAssignment.SKIP_ASSIGNMENT;
+ }
+
+ // Check if we need to unreserve something, note that we don't support
+ // continuousReservationLooking now. TODO, need think more about how to
+ // support it.
+ boolean shouldUnreserve =
+ Resources.greaterThan(rc, clusterResource,
+ resourceLimits.getAmountNeededUnreserve(), Resources.none());
+
+ // Check if we can allocate minimum resource according to headroom
+ boolean cannotAllocateAnything =
+ !checkHeadroom(clusterResource, resourceLimits, rmContext
+ .getScheduler().getMinimumResourceCapability());
+
+ // Skip the app if we failed either of above check
+ if (cannotAllocateAnything || shouldUnreserve) {
+ if (LOG.isDebugEnabled()) {
+ if (shouldUnreserve) {
+ LOG.debug("Cannot continue since we have to unreserve some resource"
+ + ", now increase container allocation doesn't "
+ + "support continuous reservation looking..");
+ }
+ if (cannotAllocateAnything) {
+ LOG.debug("We cannot allocate anything because of low headroom, "
+ + "headroom=" + resourceLimits.getHeadroom());
+ }
+ }
+
+ return CSAssignment.SKIP_ASSIGNMENT;
+ }
+
+ CSAssignment assigned = null;
+
+ /*
+ * Loop each priority, and containerId. Container priority is not
+ * equivalent to request priority, application master can run an important
+ * task on a less prioritized container.
+ *
+ * So behavior here is, we still try to increase container with higher
+ * priority, but will skip increase request and move to next increase
+ * request if queue-limit or user-limit aren't satisfied
+ */
+ for (Priority priority : application.getPriorities()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking at increase request for application="
+ + application.getApplicationAttemptId() + " priority="
+ + priority);
+ }
+
+ /*
+ * If we have multiple to-be-increased containers under same priority on
+ * a same host, we will try to increase earlier launched container
+ * first. And again - we will skip a request and move to next if it
+ * cannot be allocated.
+ */
+ Map<ContainerId, SchedContainerChangeRequest> increaseRequestMap =
+ sinfo.getIncreaseRequests(nodeId, priority);
+
+ // We don't have more increase request on this priority, skip..
+ if (null == increaseRequestMap) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("There's no increase request for "
+ + application.getApplicationAttemptId() + " priority="
+ + priority);
+ }
+ continue;
+ }
+ Iterator<Entry<ContainerId, SchedContainerChangeRequest>> iter =
+ increaseRequestMap.entrySet().iterator();
+ List<SchedContainerChangeRequest> toBeRemovedRequests =
+ new ArrayList<>();
+
+ while (iter.hasNext()) {
+ Entry<ContainerId, SchedContainerChangeRequest> entry =
+ iter.next();
+ SchedContainerChangeRequest increaseRequest =
+ entry.getValue();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Looking at increase request=" + increaseRequest.toString());
+ }
+
+ boolean headroomSatisifed = checkHeadroom(clusterResource,
+ resourceLimits, increaseRequest.getDeltaCapacity());
+ if (!headroomSatisifed) {
+ // skip if doesn't satisfy headroom limit
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" Headroom is not satisfied, skip..");
+ }
+ continue;
+ }
+
+ RMContainer rmContainer = increaseRequest.getRMContainer();
+ if (rmContainer.getContainerState() != ContainerState.RUNNING) {
+ // if the container is not running, we should remove the
+ // increaseRequest and continue;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" Container is not running any more, skip...");
+ }
+ toBeRemovedRequests.add(increaseRequest);
+ continue;
+ }
+
+ if (!Resources.fitsIn(rc, clusterResource,
+ increaseRequest.getTargetCapacity(), node.getTotalResource())) {
+ // if the target capacity is more than what the node can offer, we
+ // will simply remove and skip it.
+ // The reason of doing check here instead of adding increase request
+ // to scheduler because node's resource could be updated after
+ // request added.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" Target capacity is more than what node can offer,"
+ + " node.resource=" + node.getTotalResource());
+ }
+ toBeRemovedRequests.add(increaseRequest);
+ continue;
+ }
+
+ // Try to allocate the increase request
+ assigned =
+ allocateIncreaseRequest(node, clusterResource, increaseRequest);
+ if (!assigned.getSkipped()) {
+ // When we don't skip this request, which means we either allocated
+ // OR reserved this request. We will break
+ break;
+ }
+ }
+
+ // Remove invalid in request requests
+ if (!toBeRemovedRequests.isEmpty()) {
+ for (SchedContainerChangeRequest req : toBeRemovedRequests) {
+ sinfo.removeIncreaseRequest(req.getNodeId(), req.getPriority(),
+ req.getContainerId());
+ }
+ }
+
+ // We already allocated something
+ if (!assigned.getSkipped()) {
+ break;
+ }
+ }
+
+ return assigned == null ? CSAssignment.SKIP_ASSIGNMENT : assigned;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to allocate reserved increase container request..");
+ }
+
+ // We already reserved this increase container
+ SchedContainerChangeRequest request =
+ sinfo.getIncreaseRequest(nodeId, reservedContainer.getContainer()
+ .getPriority(), reservedContainer.getContainerId());
+
+ // We will cancel the reservation any of following happens
+ // - Container finished
+ // - No increase request needed
+ // - Target resource updated
+ if (null == request
+ || reservedContainer.getContainerState() != ContainerState.RUNNING
+ || (!Resources.equals(reservedContainer.getReservedResource(),
+ request.getDeltaCapacity()))) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("We don't need reserved increase container request "
+ + "for container=" + reservedContainer.getContainerId()
+ + ". Unreserving and return...");
+ }
+
+ // We don't need this container now, just return excessive reservation
+ return new CSAssignment(application, reservedContainer);
+ }
+
+ return allocateIncreaseRequestFromReservedContainer(node, clusterResource,
+ request);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index dcb99ed..fd99d29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
* Allocate normal (new) containers, considers locality/label, etc. Using
* delayed scheduling mechanism to get better locality allocation.
*/
-public class RegularContainerAllocator extends ContainerAllocator {
+public class RegularContainerAllocator extends AbstractContainerAllocator {
private static final Log LOG = LogFactory.getLog(RegularContainerAllocator.class);
private ResourceRequest lastResourceRequest = null;
@@ -56,6 +56,25 @@ public class RegularContainerAllocator extends ContainerAllocator {
super(application, rc, rmContext);
}
+ private boolean checkHeadroom(Resource clusterResource,
+ ResourceLimits currentResourceLimits, Resource required,
+ FiCaSchedulerNode node) {
+ // If headroom + currentReservation < required, we cannot allocate this
+ // require
+ Resource resourceCouldBeUnReserved = application.getCurrentReservation();
+ if (!application.getCSLeafQueue().getReservationContinueLooking()
+ || !node.getPartition().equals(RMNodeLabelsManager.NO_LABEL)) {
+ // If we don't allow reservation continuous looking, OR we're looking at
+ // non-default node partition, we won't allow to unreserve before
+ // allocation.
+ resourceCouldBeUnReserved = Resources.none();
+ }
+ return Resources.greaterThanOrEqual(rc, clusterResource, Resources.add(
+ currentResourceLimits.getHeadroom(), resourceCouldBeUnReserved),
+ required);
+ }
+
+
private ContainerAllocation preCheckForNewContainer(Resource clusterResource,
FiCaSchedulerNode node, SchedulingMode schedulingMode,
ResourceLimits resourceLimits, Priority priority) {
@@ -97,8 +116,9 @@ public class RegularContainerAllocator extends ContainerAllocator {
// Is the node-label-expression of this offswitch resource request
// matches the node's label?
// If not match, jump to next priority.
- if (!SchedulerUtils.checkResourceRequestMatchingNodePartition(anyRequest,
- node.getPartition(), schedulingMode)) {
+ if (!SchedulerUtils.checkResourceRequestMatchingNodePartition(
+ anyRequest.getNodeLabelExpression(), node.getPartition(),
+ schedulingMode)) {
return ContainerAllocation.PRIORITY_SKIPPED;
}
@@ -388,8 +408,8 @@ public class RegularContainerAllocator extends ContainerAllocator {
}
// check if the resource request can access the label
- if (!SchedulerUtils.checkResourceRequestMatchingNodePartition(request,
- node.getPartition(), schedulingMode)) {
+ if (!SchedulerUtils.checkResourceRequestMatchingNodePartition(
+ request.getNodeLabelExpression(), node.getPartition(), schedulingMode)) {
// this is a reserved container, but we cannot allocate it now according
// to label not match. This can be caused by node label changed
// We should un-reserve this container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 300cba9..e97da24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -58,7 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocator;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.RegularContainerAllocator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AbstractContainerAllocator;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -83,7 +84,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
private ResourceScheduler scheduler;
- private ContainerAllocator containerAllocator;
+ private AbstractContainerAllocator containerAllocator;
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
@@ -118,7 +119,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
rc = scheduler.getResourceCalculator();
}
- containerAllocator = new RegularContainerAllocator(this, rc, rmContext);
+ containerAllocator = new ContainerAllocator(this, rc, rmContext);
}
synchronized public boolean containerCompleted(RMContainer rmContainer,
@@ -207,22 +208,24 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return rmContainer;
}
- public boolean unreserve(Priority priority,
+ public synchronized boolean unreserve(Priority priority,
FiCaSchedulerNode node, RMContainer rmContainer) {
+ // Cancel increase request (if it has reserved increase request
+ rmContainer.cancelIncreaseReservation();
+
// Done with the reservation?
- if (unreserve(node, priority)) {
+ if (internalUnreserve(node, priority)) {
node.unreserveResource(this);
// Update reserved metrics
queue.getMetrics().unreserveResource(getUser(),
- rmContainer.getContainer().getResource());
+ rmContainer.getReservedResource());
return true;
}
return false;
}
- @VisibleForTesting
- public synchronized boolean unreserve(FiCaSchedulerNode node, Priority priority) {
+ private boolean internalUnreserve(FiCaSchedulerNode node, Priority priority) {
Map<NodeId, RMContainer> reservedContainers =
this.reservedContainers.get(priority);
@@ -241,7 +244,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Reset the re-reservation count
resetReReservations(priority);
- Resource resource = reservedContainer.getContainer().getResource();
+ Resource resource = reservedContainer.getReservedResource();
this.attemptResourceUsage.decReserved(node.getPartition(), resource);
LOG.info("Application " + getApplicationId() + " unreserved "
@@ -311,13 +314,15 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
ResourceRequest rr = ResourceRequest.newInstance(
Priority.UNDEFINED, ResourceRequest.ANY,
minimumAllocation, numCont);
- ContainersAndNMTokensAllocation allocation =
- pullNewlyAllocatedContainersAndNMTokens();
+ List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
+ List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
+ List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
+ List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
Resource headroom = getHeadroom();
setApplicationHeadroomForMetrics(headroom);
- return new Allocation(allocation.getContainerList(), headroom, null,
- currentContPreemption, Collections.singletonList(rr),
- allocation.getNMTokenList());
+ return new Allocation(newlyAllocatedContainers, headroom, null,
+ currentContPreemption, Collections.singletonList(rr), updatedNMTokens,
+ newlyIncreasedContainers, newlyDecreasedContainers);
}
synchronized public NodeId getNodeIdToUnreserve(Priority priority,
@@ -332,15 +337,23 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
if ((reservedContainers != null) && (!reservedContainers.isEmpty())) {
for (Map.Entry<NodeId, RMContainer> entry : reservedContainers.entrySet()) {
NodeId nodeId = entry.getKey();
- Resource containerResource = entry.getValue().getContainer().getResource();
+ RMContainer reservedContainer = entry.getValue();
+ if (reservedContainer.hasIncreaseReservation()) {
+ // Currently, only regular container allocation supports continuous
+ // reservation looking, we don't support canceling increase request
+ // reservation when allocating regular container.
+ continue;
+ }
+
+ Resource reservedResource = reservedContainer.getReservedResource();
// make sure we unreserve one with at least the same amount of
// resources, otherwise could affect capacity limits
- if (Resources.lessThanOrEqual(rc, clusterResource,
- resourceNeedUnreserve, containerResource)) {
+ if (Resources.fitsIn(rc, clusterResource, resourceNeedUnreserve,
+ reservedResource)) {
if (LOG.isDebugEnabled()) {
LOG.debug("unreserving node with reservation size: "
- + containerResource
+ + reservedResource
+ " in order to allocate container with size: " + resourceNeedUnreserve);
}
return nodeId;
@@ -374,6 +387,25 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
this.headroomProvider =
((FiCaSchedulerApp) appAttempt).getHeadroomProvider();
}
+
+ public boolean reserveIncreasedContainer(Priority priority,
+ FiCaSchedulerNode node,
+ RMContainer rmContainer, Resource reservedResource) {
+ // Inform the application
+ if (super.reserveIncreasedContainer(node, priority, rmContainer,
+ reservedResource)) {
+
+ queue.getMetrics().reserveResource(getUser(), reservedResource);
+
+ // Update the node
+ node.reserveResource(this, priority, rmContainer);
+
+ // Succeeded
+ return true;
+ }
+
+ return false;
+ }
public void reserve(Priority priority,
FiCaSchedulerNode node, RMContainer rmContainer, Container container) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a083272..56e72d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -19,7 +19,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
@@ -32,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -68,7 +76,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -914,7 +923,9 @@ public class FairScheduler extends
@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
- List<String> blacklistAdditions, List<String> blacklistRemovals) {
+ List<String> blacklistAdditions, List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
// Make sure this application exists
FSAppAttempt application = getSchedulerApp(appAttemptId);
@@ -973,18 +984,17 @@ public class FairScheduler extends
application.updateBlacklist(blacklistAdditions, blacklistRemovals);
}
- ContainersAndNMTokensAllocation allocation =
- application.pullNewlyAllocatedContainersAndNMTokens();
-
+ List<Container> newlyAllocatedContainers =
+ application.pullNewlyAllocatedContainers();
// Record container allocation time
- if (!(allocation.getContainerList().isEmpty())) {
+ if (!(newlyAllocatedContainers.isEmpty())) {
application.recordContainerAllocationTime(getClock().getTime());
}
Resource headroom = application.getHeadroom();
application.setApplicationHeadroomForMetrics(headroom);
- return new Allocation(allocation.getContainerList(), headroom,
- preemptionContainerIds, null, null, allocation.getNMTokenList());
+ return new Allocation(newlyAllocatedContainers, headroom,
+ preemptionContainerIds, null, null, application.pullUpdatedNMTokens());
}
}
@@ -1725,4 +1735,11 @@ public class FairScheduler extends
}
return targetQueueName;
}
+
+ @Override
+ protected void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest,
+ SchedulerApplicationAttempt attempt) {
+ // TODO Auto-generated method stub
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 99760df..2ec2311 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -76,7 +77,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -310,9 +311,11 @@ public class FifoScheduler extends
}
@Override
- public Allocation allocate(
- ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
- List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
+ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
+ List<ResourceRequest> ask, List<ContainerId> release,
+ List<String> blacklistAdditions, List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
if (application == null) {
LOG.error("Calling allocate on removed " +
@@ -364,12 +367,10 @@ public class FifoScheduler extends
application.updateBlacklist(blacklistAdditions, blacklistRemovals);
}
- ContainersAndNMTokensAllocation allocation =
- application.pullNewlyAllocatedContainersAndNMTokens();
Resource headroom = application.getHeadroom();
application.setApplicationHeadroomForMetrics(headroom);
- return new Allocation(allocation.getContainerList(), headroom, null,
- null, null, allocation.getNMTokenList());
+ return new Allocation(application.pullNewlyAllocatedContainers(),
+ headroom, null, null, null, application.pullUpdatedNMTokens());
}
}
@@ -1005,4 +1006,12 @@ public class FifoScheduler extends
public Resource getUsedResource() {
return usedResource;
}
+
+ @Override
+ protected void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest,
+ SchedulerApplicationAttempt attempt) {
+ // TODO Auto-generated method stub
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
index e62f7d7..b536546 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
@@ -323,7 +323,7 @@ public class Application {
// Get resources from the ResourceManager
Allocation allocation = resourceManager.getResourceScheduler().allocate(
applicationAttemptId, new ArrayList<ResourceRequest>(ask),
- new ArrayList<ContainerId>(), null, null);
+ new ArrayList<ContainerId>(), null, null, null, null);
System.out.println("-=======" + applicationAttemptId);
System.out.println("----------" + resourceManager.getRMContext().getRMApps()
.get(applicationId).getRMAppAttempt(applicationAttemptId));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index 5660b78..c325a65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -235,6 +236,14 @@ public class MockAM {
releases, null);
return allocate(req);
}
+
+ public AllocateResponse sendContainerResizingRequest(
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) throws Exception {
+ final AllocateRequest req = AllocateRequest.newInstance(0, 0F, null, null,
+ null, increaseRequests, decreaseRequests);
+ return allocate(req);
+ }
public AllocateResponse allocate(AllocateRequest allocateRequest)
throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 53cb8d0..92f3edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -19,11 +19,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -231,6 +233,17 @@ public class MockNodes {
}
return CommonNodeLabelsManager.EMPTY_STRING_SET;
}
+
+ @Override
+ public void updateNodeHeartbeatResponseForContainersDecreasing(
+ NodeHeartbeatResponse response) {
+
+ }
+
+ @Override
+ public List<Container> pullNewlyIncreasedContainers() {
+ return Collections.emptyList();
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 249f093..7ce42f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -191,6 +191,19 @@ public class MockRM extends ResourceManager {
}
}
+ public void waitForContainerState(ContainerId containerId,
+ RMContainerState state) throws Exception {
+ int timeoutSecs = 0;
+ RMContainer container = getResourceScheduler().getRMContainer(containerId);
+ while ((container == null || container.getState() != state)
+ && timeoutSecs++ < 40) {
+ System.out.println(
+ "Waiting for" + containerId + " state to be:" + state.name());
+ Thread.sleep(200);
+ }
+ Assert.assertTrue(container.getState() == state);
+ }
+
public void waitForContainerAllocated(MockNM nm, ContainerId containerId)
throws Exception {
int timeoutSecs = 0;
[22/32] hadoop git commit: YARN-1645. ContainerManager implementation
to support container resizing. Contributed by Meng Ding & Wangda Tan
Posted by wa...@apache.org.
YARN-1645. ContainerManager implementation to support container resizing. Contributed by Meng Ding & Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f13087fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f13087fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f13087fb
Branch: refs/heads/YARN-1197
Commit: f13087fb492401e4ead085643ecf687c3e5a8b79
Parents: 6694830
Author: Jian He <ji...@apache.org>
Authored: Tue Jul 21 16:10:40 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:25:28 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../CMgrDecreaseContainersResourceEvent.java | 37 ++++
.../nodemanager/ContainerManagerEventType.java | 1 +
.../containermanager/ContainerManagerImpl.java | 180 ++++++++++++++++--
.../container/ChangeContainerResourceEvent.java | 36 ++++
.../container/ContainerEventType.java | 4 +
.../nodemanager/DummyContainerManager.java | 6 +-
.../TestContainerManagerWithLCE.java | 22 +++
.../BaseContainerManagerTest.java | 43 ++++-
.../containermanager/TestContainerManager.java | 190 ++++++++++++++++++-
10 files changed, 486 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9ed0825..8eb876d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -206,6 +206,9 @@ Release 2.8.0 - UNRELEASED
YARN-1449. AM-NM protocol changes to support container resizing.
(Meng Ding & Wangda Tan via jianhe)
+ YARN-1645. ContainerManager implementation to support container resizing.
+ (Meng Ding & Wangda Tan via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
new file mode 100644
index 0000000..9479d0b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import java.util.List;
+
+public class CMgrDecreaseContainersResourceEvent extends ContainerManagerEvent {
+
+ private final List<Container> containersToDecrease;
+
+ public CMgrDecreaseContainersResourceEvent(List<Container>
+ containersToDecrease) {
+ super(ContainerManagerEventType.DECREASE_CONTAINERS_RESOURCE);
+ this.containersToDecrease = containersToDecrease;
+ }
+
+ public List<Container> getContainersToDecrease() {
+ return this.containersToDecrease;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
index 4278ce0..fcb0252 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
@@ -21,4 +21,5 @@ package org.apache.hadoop.yarn.server.nodemanager;
public enum ContainerManagerEventType {
FINISH_APPS,
FINISH_CONTAINERS,
+ DECREASE_CONTAINERS_RESOURCE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index ba1aec2..890a4e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
@@ -95,6 +96,7 @@ import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
+import org.apache.hadoop.yarn.server.nodemanager.CMgrDecreaseContainersResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -113,6 +115,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ChangeContainerResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
@@ -141,6 +144,7 @@ import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import org.apache.hadoop.yarn.util.resource.Resources;
public class ContainerManagerImpl extends CompositeService implements
ServiceStateChangeListener, ContainerManagementProtocol,
@@ -681,33 +685,45 @@ public class ContainerManagerImpl extends CompositeService implements
/**
* @param containerTokenIdentifier
- * of the container to be started
+ * of the container whose resource is to be started or increased
* @throws YarnException
*/
@Private
@VisibleForTesting
- protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
- ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
+ protected void authorizeStartAndResourceIncreaseRequest(
+ NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ boolean startRequest)
+ throws YarnException {
if (nmTokenIdentifier == null) {
throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG);
}
if (containerTokenIdentifier == null) {
throw RPCUtil.getRemoteException(INVALID_CONTAINERTOKEN_MSG);
}
+ /*
+ * Check the following:
+ * 1. The request comes from the same application attempt
+ * 2. The request possess a container token that has not expired
+ * 3. The request possess a container token that is granted by a known RM
+ */
ContainerId containerId = containerTokenIdentifier.getContainerID();
String containerIDStr = containerId.toString();
boolean unauthorized = false;
StringBuilder messageBuilder =
- new StringBuilder("Unauthorized request to start container. ");
+ new StringBuilder("Unauthorized request to " + (startRequest ?
+ "start container." : "increase container resource."));
if (!nmTokenIdentifier.getApplicationAttemptId().getApplicationId().
equals(containerId.getApplicationAttemptId().getApplicationId())) {
unauthorized = true;
messageBuilder.append("\nNMToken for application attempt : ")
.append(nmTokenIdentifier.getApplicationAttemptId())
- .append(" was used for starting container with container token")
+ .append(" was used for "
+ + (startRequest ? "starting " : "increasing resource of ")
+ + "container with container token")
.append(" issued for application attempt : ")
.append(containerId.getApplicationAttemptId());
- } else if (!this.context.getContainerTokenSecretManager()
+ } else if (startRequest && !this.context.getContainerTokenSecretManager()
.isValidStartContainerRequest(containerTokenIdentifier)) {
// Is the container being relaunched? Or RPC layer let startCall with
// tokens generated off old-secret through?
@@ -729,6 +745,14 @@ public class ContainerManagerImpl extends CompositeService implements
LOG.error(msg);
throw RPCUtil.getRemoteException(msg);
}
+ if (containerTokenIdentifier.getRMIdentifier() != nodeStatusUpdater
+ .getRMIdentifier()) {
+ // Is the container coming from unknown RM
+ StringBuilder sb = new StringBuilder("\nContainer ");
+ sb.append(containerTokenIdentifier.getContainerID().toString())
+ .append(" rejected as it is allocated by a previous RM");
+ throw new InvalidContainerException(sb.toString());
+ }
}
/**
@@ -745,7 +769,7 @@ public class ContainerManagerImpl extends CompositeService implements
}
UserGroupInformation remoteUgi = getRemoteUgi();
NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
- authorizeUser(remoteUgi,nmTokenIdentifier);
+ authorizeUser(remoteUgi, nmTokenIdentifier);
List<ContainerId> succeededContainers = new ArrayList<ContainerId>();
Map<ContainerId, SerializedException> failedContainers =
new HashMap<ContainerId, SerializedException>();
@@ -844,16 +868,8 @@ public class ContainerManagerImpl extends CompositeService implements
* belongs to correct Node Manager (part of retrieve password). c) It has
* correct RMIdentifier. d) It is not expired.
*/
- authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier);
-
- if (containerTokenIdentifier.getRMIdentifier() != nodeStatusUpdater
- .getRMIdentifier()) {
- // Is the container coming from unknown RM
- StringBuilder sb = new StringBuilder("\nContainer ");
- sb.append(containerTokenIdentifier.getContainerID().toString())
- .append(" rejected as it is allocated by a previous RM");
- throw new InvalidContainerException(sb.toString());
- }
+ authorizeStartAndResourceIncreaseRequest(
+ nmTokenIdentifier, containerTokenIdentifier, true);
// update NMToken
updateNMTokenIdentifier(nmTokenIdentifier);
@@ -960,9 +976,118 @@ public class ContainerManagerImpl extends CompositeService implements
@Override
public IncreaseContainersResourceResponse increaseContainersResource(
IncreaseContainersResourceRequest requests)
- throws YarnException, IOException {
- // To be implemented in YARN-1645
- return null;
+ throws YarnException, IOException {
+ if (blockNewContainerRequests.get()) {
+ throw new NMNotYetReadyException(
+ "Rejecting container resource increase as NodeManager has not"
+ + " yet connected with ResourceManager");
+ }
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
+ authorizeUser(remoteUgi, nmTokenIdentifier);
+ List<ContainerId> successfullyIncreasedContainers
+ = new ArrayList<ContainerId>();
+ Map<ContainerId, SerializedException> failedContainers =
+ new HashMap<ContainerId, SerializedException>();
+ // Process container resource increase requests
+ for (org.apache.hadoop.yarn.api.records.Token token :
+ requests.getContainersToIncrease()) {
+ ContainerId containerId = null;
+ try {
+ if (token.getIdentifier() == null) {
+ throw new IOException(INVALID_CONTAINERTOKEN_MSG);
+ }
+ ContainerTokenIdentifier containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(token);
+ verifyAndGetContainerTokenIdentifier(token,
+ containerTokenIdentifier);
+ authorizeStartAndResourceIncreaseRequest(
+ nmTokenIdentifier, containerTokenIdentifier, false);
+ containerId = containerTokenIdentifier.getContainerID();
+ // Reuse the startContainer logic to update NMToken,
+ // as container resource increase request will have come with
+ // an updated NMToken.
+ updateNMTokenIdentifier(nmTokenIdentifier);
+ Resource resource = containerTokenIdentifier.getResource();
+ changeContainerResourceInternal(containerId, resource, true);
+ successfullyIncreasedContainers.add(containerId);
+ } catch (YarnException | InvalidToken e) {
+ failedContainers.put(containerId, SerializedException.newInstance(e));
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+ }
+ return IncreaseContainersResourceResponse.newInstance(
+ successfullyIncreasedContainers, failedContainers);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void changeContainerResourceInternal(
+ ContainerId containerId, Resource targetResource, boolean increase)
+ throws YarnException, IOException {
+ Container container = context.getContainers().get(containerId);
+ // Check container existence
+ if (container == null) {
+ if (nodeStatusUpdater.isContainerRecentlyStopped(containerId)) {
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " was recently stopped on node manager.");
+ } else {
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " is not handled by this NodeManager");
+ }
+ }
+ // Check container state
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState currentState =
+ container.getContainerState();
+ if (currentState != org.apache.hadoop.yarn.server.
+ nodemanager.containermanager.container.ContainerState.RUNNING) {
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " is in " + currentState.name() + " state."
+ + " Resource can only be changed when a container is in"
+ + " RUNNING state");
+ }
+ // Check validity of the target resource.
+ Resource currentResource = container.getResource();
+ if (currentResource.equals(targetResource)) {
+ LOG.warn("Unable to change resource for container "
+ + containerId.toString()
+ + ". The target resource "
+ + targetResource.toString()
+ + " is the same as the current resource");
+ return;
+ }
+ if (increase && !Resources.fitsIn(currentResource, targetResource)) {
+ throw RPCUtil.getRemoteException("Unable to increase resource for "
+ + "container " + containerId.toString()
+ + ". The target resource "
+ + targetResource.toString()
+ + " is smaller than the current resource "
+ + currentResource.toString());
+ }
+ if (!increase &&
+ (!Resources.fitsIn(Resources.none(), targetResource)
+ || !Resources.fitsIn(targetResource, currentResource))) {
+ throw RPCUtil.getRemoteException("Unable to decrease resource for "
+ + "container " + containerId.toString()
+ + ". The target resource "
+ + targetResource.toString()
+ + " is not smaller than the current resource "
+ + currentResource.toString());
+ }
+ this.readLock.lock();
+ try {
+ if (!serviceStopped) {
+ dispatcher.getEventHandler().handle(new ChangeContainerResourceEvent(
+ containerId, targetResource));
+ } else {
+ throw new YarnException(
+ "Unable to change container resource as the NodeManager is "
+ + "in the process of shutting down");
+ }
+ } finally {
+ this.readLock.unlock();
+ }
}
@Private
@@ -1182,6 +1307,21 @@ public class ContainerManagerImpl extends CompositeService implements
"Container Killed by ResourceManager"));
}
break;
+ case DECREASE_CONTAINERS_RESOURCE:
+ CMgrDecreaseContainersResourceEvent containersDecreasedEvent =
+ (CMgrDecreaseContainersResourceEvent) event;
+ for (org.apache.hadoop.yarn.api.records.Container container
+ : containersDecreasedEvent.getContainersToDecrease()) {
+ try {
+ changeContainerResourceInternal(container.getId(),
+ container.getResource(), false);
+ } catch (YarnException e) {
+ LOG.error("Unable to decrease container resource", e);
+ } catch (IOException e) {
+ LOG.error("Unable to update container resource in store", e);
+ }
+ }
+ break;
default:
throw new YarnRuntimeException(
"Got an unknown ContainerManagerEvent type: " + event.getType());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java
new file mode 100644
index 0000000..3944a3d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ChangeContainerResourceEvent.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public class ChangeContainerResourceEvent extends ContainerEvent {
+
+ private Resource resource;
+
+ public ChangeContainerResourceEvent(ContainerId c, Resource resource) {
+ super(c, ContainerEventType.CHANGE_CONTAINER_RESOURCE);
+ this.resource = resource;
+ }
+
+ public Resource getResource() {
+ return this.resource;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index 5622f8c..dc712bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -25,6 +25,10 @@ public enum ContainerEventType {
KILL_CONTAINER,
UPDATE_DIAGNOSTICS_MSG,
CONTAINER_DONE,
+ CHANGE_CONTAINER_RESOURCE,
+
+ // Producer: ContainerMonitor
+ CONTAINER_RESOURCE_CHANGED,
// DownloadManager
CONTAINER_INITED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index 349340b..3ff04d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -191,8 +191,10 @@ public class DummyContainerManager extends ContainerManagerImpl {
}
@Override
- protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
- ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
+ protected void authorizeStartAndResourceIncreaseRequest(
+ NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ boolean startRequest) throws YarnException {
// do nothing
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index a47e7f7..9a05278 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -189,6 +189,28 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
super.testStartContainerFailureWithUnknownAuxService();
}
+ @Override
+ public void testIncreaseContainerResourceWithInvalidRequests() throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testIncreaseContainerResourceWithInvalidRequests");
+ super.testIncreaseContainerResourceWithInvalidRequests();
+ }
+
+ @Override
+ public void testIncreaseContainerResourceWithInvalidResource() throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testIncreaseContainerResourceWithInvalidResource");
+ super.testIncreaseContainerResourceWithInvalidResource();
+ }
+
private boolean shouldRunTest() {
return System
.getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 2810662..3938342 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -209,12 +209,13 @@ public abstract class BaseContainerManagerTest {
// do nothing
}
@Override
- protected void authorizeStartRequest(
- NMTokenIdentifier nmTokenIdentifier,
- ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
- // do nothing
- }
-
+ protected void authorizeStartAndResourceIncreaseRequest(
+ NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ boolean startRequest) throws YarnException {
+ // do nothing
+ }
+
@Override
protected void updateNMTokenIdentifier(
NMTokenIdentifier nmTokenIdentifier) throws InvalidToken {
@@ -310,4 +311,34 @@ public abstract class BaseContainerManagerTest {
app.getApplicationState().equals(finalState));
}
+ public static void waitForNMContainerState(ContainerManagerImpl
+ containerManager, ContainerId containerID,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager
+ .container.ContainerState finalState)
+ throws InterruptedException, YarnException, IOException {
+ waitForNMContainerState(containerManager, containerID, finalState, 20);
+ }
+
+ public static void waitForNMContainerState(ContainerManagerImpl
+ containerManager, ContainerId containerID,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager
+ .container.ContainerState finalState, int timeOutMax)
+ throws InterruptedException, YarnException, IOException {
+ Container container =
+ containerManager.getContext().getContainers().get(containerID);
+ org.apache.hadoop.yarn.server.nodemanager
+ .containermanager.container.ContainerState currentState =
+ container.getContainerState();
+ int timeoutSecs = 0;
+ while (!currentState.equals(finalState)
+ && timeoutSecs++ < timeOutMax) {
+ Thread.sleep(1000);
+ LOG.info("Waiting for NM container to get into state " + finalState
+ + ". Current state is " + currentState);
+ currentState = container.getContainerState();
+ }
+ LOG.info("Container state is " + currentState);
+ Assert.assertEquals("ContainerState is not correct (timedout)",
+ finalState, currentState);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f13087fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index e508424..e2f12ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -72,6 +74,7 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
+import org.apache.hadoop.yarn.server.nodemanager.CMgrDecreaseContainersResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA;
@@ -87,6 +90,8 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.junit.Assert.assertEquals;
+
public class TestContainerManager extends BaseContainerManagerTest {
public TestContainerManager() throws UnsupportedFileSystemException {
@@ -803,7 +808,8 @@ public class TestContainerManager extends BaseContainerManagerTest {
metrics, dirsHandler);
String strExceptionMsg = "";
try {
- cMgrImpl.authorizeStartRequest(null, new ContainerTokenIdentifier());
+ cMgrImpl.authorizeStartAndResourceIncreaseRequest(
+ null, new ContainerTokenIdentifier(), true);
} catch(YarnException ye) {
strExceptionMsg = ye.getMessage();
}
@@ -812,7 +818,8 @@ public class TestContainerManager extends BaseContainerManagerTest {
strExceptionMsg = "";
try {
- cMgrImpl.authorizeStartRequest(new NMTokenIdentifier(), null);
+ cMgrImpl.authorizeStartAndResourceIncreaseRequest(
+ new NMTokenIdentifier(), null, true);
} catch(YarnException ye) {
strExceptionMsg = ye.getMessage();
}
@@ -878,6 +885,167 @@ public class TestContainerManager extends BaseContainerManagerTest {
ContainerManagerImpl.INVALID_CONTAINERTOKEN_MSG);
}
+ @Test
+ public void testIncreaseContainerResourceWithInvalidRequests() throws Exception {
+ containerManager.start();
+ // Start 4 containers 0..4 with default resource (1024, 1)
+ List<StartContainerRequest> list = new ArrayList<>();
+ ContainerLaunchContext containerLaunchContext = recordFactory
+ .newRecordInstance(ContainerLaunchContext.class);
+ for (int i = 0; i < 4; i++) {
+ ContainerId cId = createContainerId(i);
+ long identifier = DUMMY_RM_IDENTIFIER;
+ Token containerToken = createContainerToken(cId, identifier,
+ context.getNodeId(), user, context.getContainerTokenSecretManager());
+ StartContainerRequest request = StartContainerRequest.newInstance(
+ containerLaunchContext, containerToken);
+ list.add(request);
+ }
+ StartContainersRequest requestList = StartContainersRequest
+ .newInstance(list);
+ StartContainersResponse response = containerManager
+ .startContainers(requestList);
+
+ Assert.assertEquals(4, response.getSuccessfullyStartedContainers().size());
+ int i = 0;
+ for (ContainerId id : response.getSuccessfullyStartedContainers()) {
+ Assert.assertEquals(i, id.getContainerId());
+ i++;
+ }
+
+ Thread.sleep(2000);
+ // Construct container resource increase request,
+ List<Token> increaseTokens = new ArrayList<Token>();
+ // Add increase request for container-0, the request will fail as the
+ // container will have exited, and won't be in RUNNING state
+ ContainerId cId0 = createContainerId(0);
+ Token containerToken =
+ createContainerToken(cId0, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user,
+ Resource.newInstance(1234, 3),
+ context.getContainerTokenSecretManager(), null);
+ increaseTokens.add(containerToken);
+ // Add increase request for container-7, the request will fail as the
+ // container does not exist
+ ContainerId cId7 = createContainerId(7);
+ containerToken =
+ createContainerToken(cId7, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user,
+ Resource.newInstance(1234, 3),
+ context.getContainerTokenSecretManager(), null);
+ increaseTokens.add(containerToken);
+
+ IncreaseContainersResourceRequest increaseRequest =
+ IncreaseContainersResourceRequest
+ .newInstance(increaseTokens);
+ IncreaseContainersResourceResponse increaseResponse =
+ containerManager.increaseContainersResource(increaseRequest);
+ // Check response
+ Assert.assertEquals(
+ 0, increaseResponse.getSuccessfullyIncreasedContainers().size());
+ Assert.assertEquals(2, increaseResponse.getFailedRequests().size());
+ for (Map.Entry<ContainerId, SerializedException> entry : increaseResponse
+ .getFailedRequests().entrySet()) {
+ Assert.assertNotNull("Failed message", entry.getValue().getMessage());
+ if (cId0.equals(entry.getKey())) {
+ Assert.assertTrue(entry.getValue().getMessage()
+ .contains("Resource can only be changed when a "
+ + "container is in RUNNING state"));
+ } else if (cId7.equals(entry.getKey())) {
+ Assert.assertTrue(entry.getValue().getMessage()
+ .contains("Container " + cId7.toString()
+ + " is not handled by this NodeManager"));
+ } else {
+ throw new YarnException("Received failed request from wrong"
+ + " container: " + entry.getKey().toString());
+ }
+ }
+ }
+
+ @Test
+ public void testIncreaseContainerResourceWithInvalidResource() throws Exception {
+ containerManager.start();
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
+ PrintWriter fileWriter = new PrintWriter(scriptFile);
+ // Construct the Container-id
+ ContainerId cId = createContainerId(0);
+ if (Shell.WINDOWS) {
+ fileWriter.println("@ping -n 100 127.0.0.1 >nul");
+ } else {
+ fileWriter.write("\numask 0");
+ fileWriter.write("\nexec sleep 100");
+ }
+ fileWriter.close();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ URL resource_alpha =
+ ConverterUtils.getYarnUrlFromPath(localFS
+ .makeQualified(new Path(scriptFile.getAbsolutePath())));
+ LocalResource rsrc_alpha =
+ recordFactory.newRecordInstance(LocalResource.class);
+ rsrc_alpha.setResource(resource_alpha);
+ rsrc_alpha.setSize(-1);
+ rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+ rsrc_alpha.setType(LocalResourceType.FILE);
+ rsrc_alpha.setTimestamp(scriptFile.lastModified());
+ String destinationFile = "dest_file";
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, rsrc_alpha);
+ containerLaunchContext.setLocalResources(localResources);
+ List<String> commands =
+ Arrays.asList(Shell.getRunScriptCommand(scriptFile));
+ containerLaunchContext.setCommands(commands);
+
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+ // Make sure the container reaches RUNNING state
+ BaseContainerManagerTest.waitForNMContainerState(containerManager, cId,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING);
+ // Construct container resource increase request,
+ List<Token> increaseTokens = new ArrayList<Token>();
+ // Add increase request. The increase request should fail
+ // as the current resource does not fit in the target resource
+ Token containerToken =
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user,
+ Resource.newInstance(512, 1),
+ context.getContainerTokenSecretManager(), null);
+ increaseTokens.add(containerToken);
+ IncreaseContainersResourceRequest increaseRequest =
+ IncreaseContainersResourceRequest
+ .newInstance(increaseTokens);
+ IncreaseContainersResourceResponse increaseResponse =
+ containerManager.increaseContainersResource(increaseRequest);
+ // Check response
+ Assert.assertEquals(
+ 0, increaseResponse.getSuccessfullyIncreasedContainers().size());
+ Assert.assertEquals(1, increaseResponse.getFailedRequests().size());
+ for (Map.Entry<ContainerId, SerializedException> entry : increaseResponse
+ .getFailedRequests().entrySet()) {
+ if (cId.equals(entry.getKey())) {
+ Assert.assertNotNull("Failed message", entry.getValue().getMessage());
+ Assert.assertTrue(entry.getValue().getMessage()
+ .contains("The target resource "
+ + Resource.newInstance(512, 1).toString()
+ + " is smaller than the current resource "
+ + Resource.newInstance(1024, 1)));
+ } else {
+ throw new YarnException("Received failed request from wrong"
+ + " container: " + entry.getKey().toString());
+ }
+ }
+ }
+
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
NodeId nodeId, String user,
NMContainerTokenSecretManager containerTokenSecretManager)
@@ -892,15 +1060,21 @@ public class TestContainerManager extends BaseContainerManagerTest {
LogAggregationContext logAggregationContext)
throws IOException {
Resource r = BuilderUtils.newResource(1024, 1);
+ return createContainerToken(cId, rmIdentifier, nodeId, user, r,
+ containerTokenSecretManager, logAggregationContext);
+ }
+
+ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
+ NodeId nodeId, String user, Resource resource,
+ NMContainerTokenSecretManager containerTokenSecretManager,
+ LogAggregationContext logAggregationContext)
+ throws IOException {
ContainerTokenIdentifier containerTokenIdentifier =
- new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
+ new ContainerTokenIdentifier(cId, nodeId.toString(), user, resource,
System.currentTimeMillis() + 100000L, 123, rmIdentifier,
Priority.newInstance(0), 0, logAggregationContext, null);
- Token containerToken =
- BuilderUtils
- .newContainerToken(nodeId, containerTokenSecretManager
- .retrievePassword(containerTokenIdentifier),
+ return BuilderUtils.newContainerToken(nodeId, containerTokenSecretManager
+ .retrievePassword(containerTokenIdentifier),
containerTokenIdentifier);
- return containerToken;
}
}
[08/32] hadoop git commit: YARN-4167. NPE on
RMActiveServices#serviceStop when store is null. (Bibin A Chundatt via
rohithsharmaks)
Posted by wa...@apache.org.
YARN-4167. NPE on RMActiveServices#serviceStop when store is null. (Bibin A Chundatt via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9cb6a59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9cb6a59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9cb6a59
Branch: refs/heads/YARN-1197
Commit: c9cb6a5960ad335a3ee93a6ee219eae5aad372f9
Parents: 3a9c707
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Sep 21 09:59:30 2015 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Sep 21 09:59:30 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9cb6a59/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7487f71..a3dfb85 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -863,6 +863,8 @@ Release 2.8.0 - UNRELEASED
YARN-4135. Improve the assertion message in MockRM while failing after waiting for the state.
(Nijel S F via rohithsharmaks)
+ YARN-4167. NPE on RMActiveServices#serviceStop when store is null. (Bibin A Chundatt via rohithsharmaks)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9cb6a59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index d6d9629..d1f339a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -605,7 +605,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
if (rmContext != null) {
RMStateStore store = rmContext.getStateStore();
try {
- store.close();
+ if (null != store) {
+ store.close();
+ }
} catch (Exception e) {
LOG.error("Error closing store.", e);
}
[32/32] hadoop git commit: YARN-4171. Fix findbugs warnings in
YARN-1197 branch. Contributed by Wangda Tan
Posted by wa...@apache.org.
YARN-4171. Fix findbugs warnings in YARN-1197 branch. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80c50a42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80c50a42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80c50a42
Branch: refs/heads/YARN-1197
Commit: 80c50a42f030ff978e66cf1d57e26659c907bb92
Parents: 874cc98
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 18 16:42:26 2015 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 22 11:26:16 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java | 1 +
.../resourcemanager/scheduler/SchedulerApplicationAttempt.java | 3 ---
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c50a42/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 196f727..92da954 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -897,6 +897,8 @@ Release 2.8.0 - UNRELEASED
YARN-4188. Make MoveApplicationAcrossQueues abstract, newInstance static.
(Giovanni Matteo Fumarola via cdouglas)
+ YARN-4171. Fix findbugs warnings in YARN-1197 branch. (Wangda Tan via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c50a42/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
index 8323f3c..0dbea1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
@@ -93,6 +93,7 @@ public class RMNodeStatusEvent extends RMNodeEvent {
this.logAggregationReportsForApps = logAggregationReportsForApps;
}
+ @SuppressWarnings("unchecked")
public List<Container> getNMReportedIncreasedContainers() {
return nmReportedIncreasedContainers == null ? Collections.EMPTY_LIST
: nmReportedIncreasedContainers;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80c50a42/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f064e97..005fa71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -478,9 +478,6 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
ContainerType containerType = ContainerType.TASK;
// The working knowledge is that masterContainer for AM is null as it
// itself is the master container.
- RMAppAttempt appAttempt = rmContext.getRMApps()
- .get(container.getId().getApplicationAttemptId().getApplicationId())
- .getCurrentAppAttempt();
if (isWaitingForAMContainer(getApplicationId())) {
containerType = ContainerType.APPLICATION_MASTER;
}
[07/32] hadoop git commit: HDFS-9004. Add upgrade domain to
DatanodeInfo. Contributed by Ming Ma (via Lei (Eddy) Xu).
Posted by wa...@apache.org.
HDFS-9004. Add upgrade domain to DatanodeInfo. Contributed by Ming Ma (via Lei (Eddy) Xu).
Change-Id: I887c66578eebd61acc34b94f18da6e6851c609f4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a9c7076
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a9c7076
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a9c7076
Branch: refs/heads/YARN-1197
Commit: 3a9c7076e81c1cc47c0ecf30c60abd9a65d8a501
Parents: c39ddc3
Author: Lei Xu <le...@apache.org>
Authored: Sat Sep 19 18:08:09 2015 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Sat Sep 19 18:08:09 2015 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 41 ++++++++++++++++++--
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 3 ++
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 3 +-
.../src/main/proto/hdfs.proto | 1 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 3 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 ++
.../server/namenode/TestNameNodeMXBean.java | 20 ++++++++++
8 files changed, 72 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index 2ef40d2..c895a1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -53,7 +53,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
private String location = NetworkTopology.DEFAULT_RACK;
private String softwareVersion;
private List<String> dependentHostNames = new LinkedList<String>();
-
+ private String upgradeDomain;
// Datanode administrative states
public enum AdminStates {
@@ -95,6 +95,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.xceiverCount = from.getXceiverCount();
this.location = from.getNetworkLocation();
this.adminState = from.getAdminState();
+ this.upgradeDomain = from.getUpgradeDomain();
}
public DatanodeInfo(DatanodeID nodeID) {
@@ -120,12 +121,13 @@ public class DatanodeInfo extends DatanodeID implements Node {
final long capacity, final long dfsUsed, final long remaining,
final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
final long lastUpdate, final long lastUpdateMonotonic,
- final int xceiverCount, final AdminStates adminState) {
+ final int xceiverCount, final AdminStates adminState,
+ final String upgradeDomain) {
this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(),
nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getInfoSecurePort(),
nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed,
cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
- xceiverCount, location, adminState);
+ xceiverCount, location, adminState, upgradeDomain);
}
/** Constructor */
@@ -137,6 +139,22 @@ public class DatanodeInfo extends DatanodeID implements Node {
final long lastUpdate, final long lastUpdateMonotonic,
final int xceiverCount, final String networkLocation,
final AdminStates adminState) {
+ this(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort, capacity, dfsUsed, remaining, blockPoolUsed, cacheCapacity,
+ cacheUsed, lastUpdate, lastUpdateMonotonic, xceiverCount,
+ networkLocation, adminState, null);
+ }
+
+ /** Constructor */
+ public DatanodeInfo(final String ipAddr, final String hostName,
+ final String datanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort,
+ final long capacity, final long dfsUsed, final long remaining,
+ final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+ final long lastUpdate, final long lastUpdateMonotonic,
+ final int xceiverCount, final String networkLocation,
+ final AdminStates adminState,
+ final String upgradeDomain) {
super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
infoSecurePort, ipcPort);
this.capacity = capacity;
@@ -150,6 +168,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.xceiverCount = xceiverCount;
this.location = networkLocation;
this.adminState = adminState;
+ this.upgradeDomain = upgradeDomain;
}
/** Network location name */
@@ -300,6 +319,16 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.location = NodeBase.normalize(location);
}
+ /** Sets the upgrade domain */
+ public void setUpgradeDomain(String upgradeDomain) {
+ this.upgradeDomain = upgradeDomain;
+ }
+
+ /** upgrade domain */
+ public String getUpgradeDomain() {
+ return upgradeDomain;
+ }
+
/** Add a hostname to a list of network dependencies */
public void addDependentHostName(String hostname) {
dependentHostNames.add(hostname);
@@ -341,6 +370,9 @@ public class DatanodeInfo extends DatanodeID implements Node {
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
buffer.append("Rack: "+location+"\n");
}
+ if (upgradeDomain != null) {
+ buffer.append("Upgrade domain: "+ upgradeDomain +"\n");
+ }
buffer.append("Decommission Status : ");
if (isDecommissioned()) {
buffer.append("Decommissioned\n");
@@ -380,6 +412,9 @@ public class DatanodeInfo extends DatanodeID implements Node {
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
buffer.append(" "+location);
}
+ if (upgradeDomain != null) {
+ buffer.append(" " + upgradeDomain);
+ }
if (isDecommissioned()) {
buffer.append(" DD");
} else if (isDecommissionInProgress()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 1e561cc..98de2e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -140,6 +140,9 @@ public class PBHelperClient {
if (info.getNetworkLocation() != null) {
builder.setLocation(info.getNetworkLocation());
}
+ if (info.getUpgradeDomain() != null) {
+ builder.setUpgradeDomain(info.getUpgradeDomain());
+ }
builder
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 713836c..3f85814 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -241,7 +241,8 @@ class JsonUtilClient {
getLong(m, "lastUpdateMonotonic", 0l),
getInt(m, "xceiverCount", 0),
getString(m, "networkLocation", ""),
- DatanodeInfo.AdminStates.valueOf(getString(m, "adminState", "NORMAL")));
+ DatanodeInfo.AdminStates.valueOf(getString(m, "adminState", "NORMAL")),
+ getString(m, "upgradeDomain", ""));
}
/** Convert an Object[] to a DatanodeInfo[]. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 86fb462..ee77dc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -98,6 +98,7 @@ message DatanodeInfoProto {
optional uint64 cacheCapacity = 11 [default = 0];
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
+ optional string upgradeDomain = 14;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 5b60307..cf55445 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -594,7 +594,8 @@ public class PBHelper {
di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), PBHelper.convert(di.getAdminState()));
+ di.getXceiverCount(), PBHelper.convert(di.getAdminState()),
+ di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
}
static public DatanodeInfo[] convert(DatanodeInfoProto di[]) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 328c29d..75b6be9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5922,6 +5922,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.put("estimatedCapacityLostTotal",
volumeFailureSummary.getEstimatedCapacityLostTotal());
}
+ if (node.getUpgradeDomain() != null) {
+ innerinfo.put("upgradeDomain", node.getUpgradeDomain());
+ }
info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo.build());
}
return JSON.toString(info);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 0de89cc..4b0e63e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -169,6 +169,9 @@ public class JsonUtil {
m.put("xceiverCount", datanodeinfo.getXceiverCount());
m.put("networkLocation", datanodeinfo.getNetworkLocation());
m.put("adminState", datanodeinfo.getAdminState().name());
+ if (datanodeinfo.getUpgradeDomain() != null) {
+ m.put("upgradeDomain", datanodeinfo.getUpgradeDomain());
+ }
return m;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a9c7076/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index f8aa317..f10d57e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -76,6 +78,15 @@ public class TestNameNodeMXBean {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
+ // Set upgrade domain on the first DN.
+ String upgradeDomain = "abcd";
+ DatanodeManager dm = cluster.getNameNode().getNamesystem().
+ getBlockManager().getDatanodeManager();
+ DatanodeDescriptor dd = dm.getDatanode(
+ cluster.getDataNodes().get(0).getDatanodeId());
+ dd.setUpgradeDomain(upgradeDomain);
+ String dnXferAddrWithUpgradeDomainSet = dd.getXferAddr();
+
FSNamesystem fsn = cluster.getNameNode().namesystem;
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
@@ -125,6 +136,15 @@ public class TestNameNodeMXBean {
assertTrue(((Long)liveNode.get("capacity")) > 0);
assertTrue(liveNode.containsKey("numBlocks"));
assertTrue(((Long)liveNode.get("numBlocks")) == 0);
+ // a. By default the upgrade domain isn't defined on any DN.
+ // b. If the upgrade domain is set on a DN, JMX should have the same
+ // value.
+ String xferAddr = (String)liveNode.get("xferaddr");
+ if (!xferAddr.equals(dnXferAddrWithUpgradeDomainSet)) {
+ assertTrue(!liveNode.containsKey("upgradeDomain"));
+ } else {
+ assertTrue(liveNode.get("upgradeDomain").equals(upgradeDomain));
+ }
}
assertEquals(fsn.getLiveNodes(), alivenodeinfo);
// get attribute deadnodeinfo
[30/32] hadoop git commit: YARN-1651. CapacityScheduler side changes
to support container resize. Contributed by Wangda Tan
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 27d70cc..6a4efa1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -19,7 +19,16 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -37,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -51,6 +61,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -58,13 +69,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRecoverEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecreaseContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
@@ -87,7 +100,7 @@ public abstract class AbstractYarnScheduler
protected Resource clusterResource = Resource.newInstance(0, 0);
protected Resource minimumAllocation;
- private Resource maximumAllocation;
+ protected Resource maximumAllocation;
private Resource configuredMaximumAllocation;
private int maxNodeMemory = -1;
private int maxNodeVCores = -1;
@@ -231,6 +244,55 @@ public abstract class AbstractYarnScheduler
application.containerLaunchedOnNode(containerId, node.getNodeID());
}
+
+ protected synchronized void containerIncreasedOnNode(ContainerId containerId,
+ SchedulerNode node, Container increasedContainerReportedByNM) {
+ // Get the application for the finished container
+ SchedulerApplicationAttempt application =
+ getCurrentAttemptForContainer(containerId);
+ if (application == null) {
+ LOG.info("Unknown application "
+ + containerId.getApplicationAttemptId().getApplicationId()
+ + " increased container " + containerId + " on node: " + node);
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));
+ return;
+ }
+
+ RMContainer rmContainer = getRMContainer(containerId);
+ Resource rmContainerResource = rmContainer.getAllocatedResource();
+ Resource nmContainerResource = increasedContainerReportedByNM.getResource();
+
+
+ if (Resources.equals(nmContainerResource, rmContainerResource)){
+ // NM reported expected container size, tell RMContainer. Which will stop
+ // container expire monitor
+ rmContainer.handle(new RMContainerEvent(containerId,
+ RMContainerEventType.NM_DONE_CHANGE_RESOURCE));
+ } else if (Resources.fitsIn(getResourceCalculator(), clusterResource,
+ nmContainerResource, rmContainerResource)) {
+ // when rmContainerResource >= nmContainerResource, we won't do anything,
+ // it is possible a container increased is issued by RM, but AM hasn't
+ // told NM.
+ } else if (Resources.fitsIn(getResourceCalculator(), clusterResource,
+ rmContainerResource, nmContainerResource)) {
+ // When rmContainerResource <= nmContainerResource, it could happen when a
+ // container decreased by RM before it is increased in NM.
+
+ // Tell NM to decrease the container
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeDecreaseContainerEvent(node.getNodeID(),
+ Arrays.asList(rmContainer.getContainer())));
+ } else {
+ // Something wrong happened, kill the container
+ LOG.warn("Something wrong happened, container size reported by NM"
+ + " is not expected, ContainerID=" + containerId
+ + " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:"
+ + nmContainerResource);
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));
+ }
+ }
public T getApplicationAttempt(ApplicationAttemptId applicationAttemptId) {
SchedulerApplication<T> app =
@@ -511,6 +573,36 @@ public abstract class AbstractYarnScheduler
SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED);
}
}
+
+ protected void decreaseContainers(
+ List<SchedContainerChangeRequest> decreaseRequests,
+ SchedulerApplicationAttempt attempt) {
+ for (SchedContainerChangeRequest request : decreaseRequests) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing decrease request:" + request);
+ }
+
+ boolean hasIncreaseRequest =
+ attempt.removeIncreaseRequest(request.getNodeId(),
+ request.getPriority(), request.getContainerId());
+
+ if (hasIncreaseRequest) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("While processing decrease request, found a increase request "
+ + "for the same container "
+ + request.getContainerId()
+ + ", removed the increase request");
+ }
+ }
+
+ // handle decrease request
+ decreaseContainer(request, attempt);
+ }
+ }
+
+ protected abstract void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest,
+ SchedulerApplicationAttempt attempt);
public SchedulerNode getSchedulerNode(NodeId nodeId) {
return nodes.get(nodeId);
@@ -735,4 +827,56 @@ public abstract class AbstractYarnScheduler
LOG.info("Updated the cluste max priority to maxClusterLevelAppPriority = "
+ maxClusterLevelAppPriority);
}
+
+ /**
+ * Normalize container increase/decrease request, and return
+ * SchedulerContainerResourceChangeRequest according to given
+ * ContainerResourceChangeRequest.
+ *
+ * <pre>
+ * - Returns non-null value means validation succeeded
+ * - Throw exception when any other error happens
+ * </pre>
+ */
+ private SchedContainerChangeRequest
+ checkAndNormalizeContainerChangeRequest(
+ ContainerResourceChangeRequest request, boolean increase)
+ throws YarnException {
+ // We have done a check in ApplicationMasterService, but RMContainer status
+ // / Node resource could change since AMS won't acquire lock of scheduler.
+ RMServerUtils.checkAndNormalizeContainerChangeRequest(rmContext, request,
+ increase);
+ ContainerId containerId = request.getContainerId();
+ RMContainer rmContainer = getRMContainer(containerId);
+ SchedulerNode schedulerNode =
+ getSchedulerNode(rmContainer.getAllocatedNode());
+
+ return new SchedContainerChangeRequest(schedulerNode, rmContainer,
+ request.getCapability());
+ }
+
+ protected List<SchedContainerChangeRequest>
+ checkAndNormalizeContainerChangeRequests(
+ List<ContainerResourceChangeRequest> changeRequests,
+ boolean increase) {
+ if (null == changeRequests || changeRequests.isEmpty()) {
+ return Collections.EMPTY_LIST;
+ }
+
+ List<SchedContainerChangeRequest> schedulerChangeRequests =
+ new ArrayList<SchedContainerChangeRequest>();
+ for (ContainerResourceChangeRequest r : changeRequests) {
+ SchedContainerChangeRequest sr = null;
+ try {
+ sr = checkAndNormalizeContainerChangeRequest(r, increase);
+ } catch (YarnException e) {
+ LOG.warn("Error happens when checking increase request, Ignoring.."
+ + " exception=", e);
+ continue;
+ }
+ schedulerChangeRequests.add(sr);
+ }
+
+ return schedulerChangeRequests;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
index 3f2d8af..af6caad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Allocation.java
@@ -34,6 +34,9 @@ public class Allocation {
final Set<ContainerId> fungibleContainers;
final List<ResourceRequest> fungibleResources;
final List<NMToken> nmTokens;
+ final List<Container> increasedContainers;
+ final List<Container> decreasedContainers;
+
public Allocation(List<Container> containers, Resource resourceLimit,
Set<ContainerId> strictContainers, Set<ContainerId> fungibleContainers,
@@ -45,12 +48,22 @@ public class Allocation {
public Allocation(List<Container> containers, Resource resourceLimit,
Set<ContainerId> strictContainers, Set<ContainerId> fungibleContainers,
List<ResourceRequest> fungibleResources, List<NMToken> nmTokens) {
+ this(containers, resourceLimit,strictContainers, fungibleContainers,
+ fungibleResources, nmTokens, null, null);
+ }
+
+ public Allocation(List<Container> containers, Resource resourceLimit,
+ Set<ContainerId> strictContainers, Set<ContainerId> fungibleContainers,
+ List<ResourceRequest> fungibleResources, List<NMToken> nmTokens,
+ List<Container> increasedContainers, List<Container> decreasedContainer) {
this.containers = containers;
this.resourceLimit = resourceLimit;
this.strictContainers = strictContainers;
this.fungibleContainers = fungibleContainers;
this.fungibleResources = fungibleResources;
this.nmTokens = nmTokens;
+ this.increasedContainers = increasedContainers;
+ this.decreasedContainers = decreasedContainer;
}
public List<Container> getContainers() {
@@ -76,5 +89,12 @@ public class Allocation {
public List<NMToken> getNMTokens() {
return nmTokens;
}
-
+
+ public List<Container> getIncreasedContainers() {
+ return increasedContainers;
+ }
+
+ public List<Container> getDecreasedContainers() {
+ return decreasedContainers;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index e318d47..7623da0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
@@ -35,6 +37,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -63,8 +67,11 @@ public class AppSchedulingInfo {
final Set<Priority> priorities = new TreeSet<Priority>(
new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator());
- final Map<Priority, Map<String, ResourceRequest>> requests =
- new ConcurrentHashMap<Priority, Map<String, ResourceRequest>>();
+ final Map<Priority, Map<String, ResourceRequest>> resourceRequestMap =
+ new ConcurrentHashMap<Priority, Map<String, ResourceRequest>>();
+ final Map<NodeId, Map<Priority, Map<ContainerId,
+ SchedContainerChangeRequest>>> increaseRequestMap =
+ new ConcurrentHashMap<>();
private Set<String> userBlacklist = new HashSet<>();
private Set<String> amBlacklist = new HashSet<>();
@@ -114,13 +121,177 @@ public class AppSchedulingInfo {
*/
private synchronized void clearRequests() {
priorities.clear();
- requests.clear();
+ resourceRequestMap.clear();
LOG.info("Application " + applicationId + " requests cleared");
}
public long getNewContainerId() {
return this.containerIdCounter.incrementAndGet();
}
+
+ public boolean hasIncreaseRequest(NodeId nodeId) {
+ Map<Priority, Map<ContainerId, SchedContainerChangeRequest>> requestsOnNode =
+ increaseRequestMap.get(nodeId);
+ if (null == requestsOnNode) {
+ return false;
+ }
+ return requestsOnNode.size() > 0;
+ }
+
+ public Map<ContainerId, SchedContainerChangeRequest>
+ getIncreaseRequests(NodeId nodeId, Priority priority) {
+ Map<Priority, Map<ContainerId, SchedContainerChangeRequest>> requestsOnNode =
+ increaseRequestMap.get(nodeId);
+ if (null == requestsOnNode) {
+ return null;
+ }
+
+ return requestsOnNode.get(priority);
+ }
+
+ public synchronized boolean updateIncreaseRequests(
+ List<SchedContainerChangeRequest> increaseRequests) {
+ boolean resourceUpdated = false;
+
+ for (SchedContainerChangeRequest r : increaseRequests) {
+ NodeId nodeId = r.getRMContainer().getAllocatedNode();
+
+ Map<Priority, Map<ContainerId, SchedContainerChangeRequest>> requestsOnNode =
+ increaseRequestMap.get(nodeId);
+ if (null == requestsOnNode) {
+ requestsOnNode = new TreeMap<>();
+ increaseRequestMap.put(nodeId, requestsOnNode);
+ }
+
+ SchedContainerChangeRequest prevChangeRequest =
+ getIncreaseRequest(nodeId, r.getPriority(), r.getContainerId());
+ if (null != prevChangeRequest) {
+ if (Resources.equals(prevChangeRequest.getTargetCapacity(),
+ r.getTargetCapacity())) {
+ // New target capacity is as same as what we have, just ignore the new
+ // one
+ continue;
+ }
+
+ // remove the old one
+ removeIncreaseRequest(nodeId, prevChangeRequest.getPriority(),
+ prevChangeRequest.getContainerId());
+ }
+
+ if (Resources.equals(r.getTargetCapacity(), r.getRMContainer().getAllocatedResource())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to increase/decrease container, "
+ + "target capacity = previous capacity = " + prevChangeRequest
+ + " for container=" + r.getContainerId()
+ + ". Will ignore this increase request");
+ }
+ continue;
+ }
+
+ // add the new one
+ resourceUpdated = true;
+ insertIncreaseRequest(r);
+ }
+ return resourceUpdated;
+ }
+
+ // insert increase request and add missing hierarchy if missing
+ private void insertIncreaseRequest(SchedContainerChangeRequest request) {
+ NodeId nodeId = request.getNodeId();
+ Priority priority = request.getPriority();
+ ContainerId containerId = request.getContainerId();
+
+ Map<Priority, Map<ContainerId, SchedContainerChangeRequest>> requestsOnNode =
+ increaseRequestMap.get(nodeId);
+ if (null == requestsOnNode) {
+ requestsOnNode =
+ new HashMap<Priority, Map<ContainerId, SchedContainerChangeRequest>>();
+ increaseRequestMap.put(nodeId, requestsOnNode);
+ }
+
+ Map<ContainerId, SchedContainerChangeRequest> requestsOnNodeWithPriority =
+ requestsOnNode.get(priority);
+ if (null == requestsOnNodeWithPriority) {
+ requestsOnNodeWithPriority =
+ new TreeMap<ContainerId, SchedContainerChangeRequest>();
+ requestsOnNode.put(priority, requestsOnNodeWithPriority);
+ }
+
+ requestsOnNodeWithPriority.put(containerId, request);
+
+ // update resources
+ String partition = request.getRMContainer().getNodeLabelExpression();
+ Resource delta = request.getDeltaCapacity();
+ appResourceUsage.incPending(partition, delta);
+ queue.incPendingResource(partition, delta);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Added increase request:" + request.getContainerId()
+ + " delta=" + request.getDeltaCapacity());
+ }
+
+ // update priorities
+ priorities.add(priority);
+ }
+
+ public synchronized boolean removeIncreaseRequest(NodeId nodeId, Priority priority,
+ ContainerId containerId) {
+ Map<Priority, Map<ContainerId, SchedContainerChangeRequest>> requestsOnNode =
+ increaseRequestMap.get(nodeId);
+ if (null == requestsOnNode) {
+ return false;
+ }
+
+ Map<ContainerId, SchedContainerChangeRequest> requestsOnNodeWithPriority =
+ requestsOnNode.get(priority);
+ if (null == requestsOnNodeWithPriority) {
+ return false;
+ }
+
+ SchedContainerChangeRequest request =
+ requestsOnNodeWithPriority.remove(containerId);
+
+ // remove hierarchies if it becomes empty
+ if (requestsOnNodeWithPriority.isEmpty()) {
+ requestsOnNode.remove(priority);
+ }
+ if (requestsOnNode.isEmpty()) {
+ increaseRequestMap.remove(nodeId);
+ }
+
+ if (request == null) {
+ return false;
+ }
+
+ // update queue's pending resource if request exists
+ String partition = request.getRMContainer().getNodeLabelExpression();
+ Resource delta = request.getDeltaCapacity();
+ appResourceUsage.decPending(partition, delta);
+ queue.decPendingResource(partition, delta);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("remove increase request:" + request);
+ }
+
+ return true;
+ }
+
+ public SchedContainerChangeRequest getIncreaseRequest(NodeId nodeId,
+ Priority priority, ContainerId containerId) {
+ Map<Priority, Map<ContainerId, SchedContainerChangeRequest>> requestsOnNode =
+ increaseRequestMap.get(nodeId);
+ if (null == requestsOnNode) {
+ return null;
+ }
+
+ Map<ContainerId, SchedContainerChangeRequest> requestsOnNodeWithPriority =
+ requestsOnNode.get(priority);
+ if (null == requestsOnNodeWithPriority) {
+ return null;
+ }
+
+ return requestsOnNodeWithPriority.get(containerId);
+ }
/**
* The ApplicationMaster is updating resource requirements for the
@@ -163,11 +334,11 @@ public class AppSchedulingInfo {
}
}
- Map<String, ResourceRequest> asks = this.requests.get(priority);
+ Map<String, ResourceRequest> asks = this.resourceRequestMap.get(priority);
if (asks == null) {
asks = new ConcurrentHashMap<String, ResourceRequest>();
- this.requests.put(priority, asks);
+ this.resourceRequestMap.put(priority, asks);
this.priorities.add(priority);
}
lastRequest = asks.get(resourceName);
@@ -260,12 +431,12 @@ public class AppSchedulingInfo {
synchronized public Map<String, ResourceRequest> getResourceRequests(
Priority priority) {
- return requests.get(priority);
+ return resourceRequestMap.get(priority);
}
public List<ResourceRequest> getAllResourceRequests() {
List<ResourceRequest> ret = new ArrayList<ResourceRequest>();
- for (Map<String, ResourceRequest> r : requests.values()) {
+ for (Map<String, ResourceRequest> r : resourceRequestMap.values()) {
ret.addAll(r.values());
}
return ret;
@@ -273,7 +444,7 @@ public class AppSchedulingInfo {
synchronized public ResourceRequest getResourceRequest(Priority priority,
String resourceName) {
- Map<String, ResourceRequest> nodeRequests = requests.get(priority);
+ Map<String, ResourceRequest> nodeRequests = resourceRequestMap.get(priority);
return (nodeRequests == null) ? null : nodeRequests.get(resourceName);
}
@@ -301,6 +472,50 @@ public class AppSchedulingInfo {
}
}
+ public synchronized void increaseContainer(
+ SchedContainerChangeRequest increaseRequest) {
+ NodeId nodeId = increaseRequest.getNodeId();
+ Priority priority = increaseRequest.getPriority();
+ ContainerId containerId = increaseRequest.getContainerId();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("allocated increase request : applicationId=" + applicationId
+ + " container=" + containerId + " host="
+ + increaseRequest.getNodeId() + " user=" + user + " resource="
+ + increaseRequest.getDeltaCapacity());
+ }
+
+ // Set queue metrics
+ queue.getMetrics().allocateResources(user, 0,
+ increaseRequest.getDeltaCapacity(), true);
+
+ // remove the increase request from pending increase request map
+ removeIncreaseRequest(nodeId, priority, containerId);
+
+ // update usage
+ appResourceUsage.incUsed(increaseRequest.getNodePartition(),
+ increaseRequest.getDeltaCapacity());
+ }
+
+ public synchronized void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest) {
+ // Delta is negative when it's a decrease request
+ Resource absDelta = Resources.negate(decreaseRequest.getDeltaCapacity());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Decrease container : applicationId=" + applicationId
+ + " container=" + decreaseRequest.getContainerId() + " host="
+ + decreaseRequest.getNodeId() + " user=" + user + " resource="
+ + absDelta);
+ }
+
+ // Set queue metrics
+ queue.getMetrics().releaseResources(user, 0, absDelta);
+
+ // update usage
+ appResourceUsage.decUsed(decreaseRequest.getNodePartition(), absDelta);
+ }
+
/**
* Resources have been allocated to this application by the resource
* scheduler. Track them.
@@ -359,11 +574,11 @@ public class AppSchedulingInfo {
// Update future requirements
decResourceRequest(node.getNodeName(), priority, nodeLocalRequest);
- ResourceRequest rackLocalRequest = requests.get(priority).get(
+ ResourceRequest rackLocalRequest = resourceRequestMap.get(priority).get(
node.getRackName());
decResourceRequest(node.getRackName(), priority, rackLocalRequest);
- ResourceRequest offRackRequest = requests.get(priority).get(
+ ResourceRequest offRackRequest = resourceRequestMap.get(priority).get(
ResourceRequest.ANY);
decrementOutstanding(offRackRequest);
@@ -377,7 +592,7 @@ public class AppSchedulingInfo {
ResourceRequest request) {
request.setNumContainers(request.getNumContainers() - 1);
if (request.getNumContainers() == 0) {
- requests.get(priority).remove(resourceName);
+ resourceRequestMap.get(priority).remove(resourceName);
}
}
@@ -394,7 +609,7 @@ public class AppSchedulingInfo {
// Update future requirements
decResourceRequest(node.getRackName(), priority, rackLocalRequest);
- ResourceRequest offRackRequest = requests.get(priority).get(
+ ResourceRequest offRackRequest = resourceRequestMap.get(priority).get(
ResourceRequest.ANY);
decrementOutstanding(offRackRequest);
@@ -449,6 +664,12 @@ public class AppSchedulingInfo {
}
}
}
+
+ // also we need to check increase request
+ if (!deactivate) {
+ deactivate = increaseRequestMap.isEmpty();
+ }
+
if (deactivate) {
activeUsersManager.deactivateApplication(user, applicationId);
}
@@ -457,7 +678,7 @@ public class AppSchedulingInfo {
synchronized public void move(Queue newQueue) {
QueueMetrics oldMetrics = queue.getMetrics();
QueueMetrics newMetrics = newQueue.getMetrics();
- for (Map<String, ResourceRequest> asks : requests.values()) {
+ for (Map<String, ResourceRequest> asks : resourceRequestMap.values()) {
ResourceRequest request = asks.get(ResourceRequest.ANY);
if (request != null) {
oldMetrics.decrPendingResources(user, request.getNumContainers(),
@@ -484,7 +705,7 @@ public class AppSchedulingInfo {
synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) {
// clear pending resources metrics for the application
QueueMetrics metrics = queue.getMetrics();
- for (Map<String, ResourceRequest> asks : requests.values()) {
+ for (Map<String, ResourceRequest> asks : resourceRequestMap.values()) {
ResourceRequest request = asks.get(ResourceRequest.ANY);
if (request != null) {
metrics.decrPendingResources(user, request.getNumContainers(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index 09fd73e..d94b621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -373,17 +373,20 @@ public class QueueMetrics implements MetricsSource {
}
private void _decrPendingResources(int containers, Resource res) {
+ // if #container = 0, means change container resource
pendingContainers.decr(containers);
- pendingMB.decr(res.getMemory() * containers);
- pendingVCores.decr(res.getVirtualCores() * containers);
+ pendingMB.decr(res.getMemory() * Math.max(containers, 1));
+ pendingVCores.decr(res.getVirtualCores() * Math.max(containers, 1));
}
public void allocateResources(String user, int containers, Resource res,
boolean decrPending) {
+ // if #containers = 0, means change container resource
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
- allocatedMB.incr(res.getMemory() * containers);
- allocatedVCores.incr(res.getVirtualCores() * containers);
+
+ allocatedMB.incr(res.getMemory() * Math.max(containers, 1));
+ allocatedVCores.incr(res.getVirtualCores() * Math.max(containers, 1));
if (decrPending) {
_decrPendingResources(containers, res);
}
@@ -397,10 +400,11 @@ public class QueueMetrics implements MetricsSource {
}
public void releaseResources(String user, int containers, Resource res) {
+ // if #container = 0, means change container resource.
allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers);
- allocatedMB.decr(res.getMemory() * containers);
- allocatedVCores.decr(res.getVirtualCores() * containers);
+ allocatedMB.decr(res.getMemory() * Math.max(containers, 1));
+ allocatedVCores.decr(res.getVirtualCores() * Math.max(containers, 1));
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.releaseResources(user, containers, res);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedContainerChangeRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedContainerChangeRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedContainerChangeRequest.java
new file mode 100644
index 0000000..ea109fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedContainerChangeRequest.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This is ContainerResourceChangeRequest in scheduler side, it contains some
+ * pointers to runtime objects like RMContainer, SchedulerNode, etc. This will
+ * be easier for scheduler making decision.
+ */
+public class SchedContainerChangeRequest implements
+ Comparable<SchedContainerChangeRequest> {
+ RMContainer rmContainer;
+ Resource targetCapacity;
+ SchedulerNode schedulerNode;
+ Resource deltaCapacity;
+
+ public SchedContainerChangeRequest(SchedulerNode schedulerNode,
+ RMContainer rmContainer, Resource targetCapacity) {
+ this.rmContainer = rmContainer;
+ this.targetCapacity = targetCapacity;
+ this.schedulerNode = schedulerNode;
+ deltaCapacity = Resources.subtract(targetCapacity,
+ rmContainer.getAllocatedResource());
+ }
+
+ public NodeId getNodeId() {
+ return this.rmContainer.getAllocatedNode();
+ }
+
+ public RMContainer getRMContainer() {
+ return this.rmContainer;
+ }
+
+ public Resource getTargetCapacity() {
+ return this.targetCapacity;
+ }
+
+ /**
+ * Delta capacity = before - target, so if it is a decrease request, delta
+ * capacity will be negative
+ */
+ public Resource getDeltaCapacity() {
+ return deltaCapacity;
+ }
+
+ public Priority getPriority() {
+ return rmContainer.getContainer().getPriority();
+ }
+
+ public ContainerId getContainerId() {
+ return rmContainer.getContainerId();
+ }
+
+ public String getNodePartition() {
+ return schedulerNode.getPartition();
+ }
+
+ public SchedulerNode getSchedulerNode() {
+ return schedulerNode;
+ }
+
+ @Override
+ public int hashCode() {
+ return (getContainerId().hashCode() << 16) + targetCapacity.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof SchedContainerChangeRequest)) {
+ return false;
+ }
+ return compareTo((SchedContainerChangeRequest)other) == 0;
+ }
+
+ @Override
+ public int compareTo(SchedContainerChangeRequest other) {
+ if (other == null) {
+ return -1;
+ }
+
+ int rc = getPriority().compareTo(other.getPriority());
+ if (0 != rc) {
+ return rc;
+ }
+
+ return getContainerId().compareTo(other.getContainerId());
+ }
+
+ @Override
+ public String toString() {
+ return "<container=" + getContainerId() + ", targetCapacity="
+ + targetCapacity + ", delta=" + deltaCapacity + ", node="
+ + getNodeId().toString() + ">";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
index 519de98..96288f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
@@ -28,7 +28,7 @@ public class SchedulerApplication<T extends SchedulerApplicationAttempt> {
private Queue queue;
private final String user;
- private T currentAttempt;
+ private volatile T currentAttempt;
private volatile Priority priority;
public SchedulerApplication(Queue queue, String user) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index b361d15..f064e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -19,11 +19,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -51,16 +53,19 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppR
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerChangeResourceEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerUpdatesAcquiredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -104,8 +109,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private AtomicLong firstAllocationRequestSentTime = new AtomicLong(0);
private AtomicLong firstContainerAllocatedTime = new AtomicLong(0);
- protected List<RMContainer> newlyAllocatedContainers =
- new ArrayList<RMContainer>();
+ protected List<RMContainer> newlyAllocatedContainers = new ArrayList<>();
+ protected Map<ContainerId, RMContainer> newlyDecreasedContainers = new HashMap<>();
+ protected Map<ContainerId, RMContainer> newlyIncreasedContainers = new HashMap<>();
+ protected Set<NMToken> updatedNMTokens = new HashSet<>();
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
@@ -219,7 +226,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return appSchedulingInfo.getPriorities();
}
- public synchronized ResourceRequest getResourceRequest(Priority priority, String resourceName) {
+ public synchronized ResourceRequest getResourceRequest(Priority priority,
+ String resourceName) {
return this.appSchedulingInfo.getResourceRequest(priority, resourceName);
}
@@ -324,24 +332,28 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return reservedContainers;
}
- public synchronized RMContainer reserve(SchedulerNode node, Priority priority,
- RMContainer rmContainer, Container container) {
- // Create RMContainer if necessary
- if (rmContainer == null) {
- rmContainer =
- new RMContainerImpl(container, getApplicationAttemptId(),
- node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
+ public synchronized boolean reserveIncreasedContainer(SchedulerNode node,
+ Priority priority, RMContainer rmContainer, Resource reservedResource) {
+ if (commonReserve(node, priority, rmContainer, reservedResource)) {
attemptResourceUsage.incReserved(node.getPartition(),
- container.getResource());
-
- // Reset the re-reservation count
- resetReReservations(priority);
- } else {
- // Note down the re-reservation
- addReReservation(priority);
+ reservedResource);
+ // succeeded
+ return true;
+ }
+
+ return false;
+ }
+
+ private synchronized boolean commonReserve(SchedulerNode node,
+ Priority priority, RMContainer rmContainer, Resource reservedResource) {
+ try {
+ rmContainer.handle(new RMContainerReservedEvent(rmContainer
+ .getContainerId(), reservedResource, node.getNodeID(), priority));
+ } catch (InvalidStateTransitionException e) {
+ // We reach here could be caused by container already finished, return
+ // false indicate it fails
+ return false;
}
- rmContainer.handle(new RMContainerReservedEvent(container.getId(),
- container.getResource(), node.getNodeID(), priority));
Map<NodeId, RMContainer> reservedContainers =
this.reservedContainers.get(priority);
@@ -356,8 +368,30 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
+ " reserved container " + rmContainer + " on node " + node
+ ". This attempt currently has " + reservedContainers.size()
+ " reserved containers at priority " + priority
- + "; currentReservation " + container.getResource());
+ + "; currentReservation " + reservedResource);
}
+
+ return true;
+ }
+
+ public synchronized RMContainer reserve(SchedulerNode node,
+ Priority priority, RMContainer rmContainer, Container container) {
+ // Create RMContainer if necessary
+ if (rmContainer == null) {
+ rmContainer =
+ new RMContainerImpl(container, getApplicationAttemptId(),
+ node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
+ attemptResourceUsage.incReserved(node.getPartition(),
+ container.getResource());
+
+ // Reset the re-reservation count
+ resetReReservations(priority);
+ } else {
+ // Note down the re-reservation
+ addReReservation(priority);
+ }
+
+ commonReserve(node, priority, rmContainer, container.getResource());
return rmContainer;
}
@@ -437,69 +471,100 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
public Resource getCurrentConsumption() {
return attemptResourceUsage.getUsed();
}
-
- public static class ContainersAndNMTokensAllocation {
- List<Container> containerList;
- List<NMToken> nmTokenList;
-
- public ContainersAndNMTokensAllocation(List<Container> containerList,
- List<NMToken> nmTokenList) {
- this.containerList = containerList;
- this.nmTokenList = nmTokenList;
+
+ private Container updateContainerAndNMToken(RMContainer rmContainer,
+ boolean newContainer, boolean increasedContainer) {
+ Container container = rmContainer.getContainer();
+ ContainerType containerType = ContainerType.TASK;
+ // The working knowledge is that masterContainer for AM is null as it
+ // itself is the master container.
+ RMAppAttempt appAttempt = rmContext.getRMApps()
+ .get(container.getId().getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
+ if (isWaitingForAMContainer(getApplicationId())) {
+ containerType = ContainerType.APPLICATION_MASTER;
}
-
- public List<Container> getContainerList() {
- return containerList;
+ try {
+ // create container token and NMToken altogether.
+ container.setContainerToken(rmContext.getContainerTokenSecretManager()
+ .createContainerToken(container.getId(), container.getNodeId(),
+ getUser(), container.getResource(), container.getPriority(),
+ rmContainer.getCreationTime(), this.logAggregationContext,
+ rmContainer.getNodeLabelExpression(), containerType));
+ NMToken nmToken =
+ rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
+ getApplicationAttemptId(), container);
+ if (nmToken != null) {
+ updatedNMTokens.add(nmToken);
+ }
+ } catch (IllegalArgumentException e) {
+ // DNS might be down, skip returning this container.
+ LOG.error("Error trying to assign container token and NM token to"
+ + " an updated container " + container.getId(), e);
+ return null;
}
-
- public List<NMToken> getNMTokenList() {
- return nmTokenList;
+
+ if (newContainer) {
+ rmContainer.handle(new RMContainerEvent(
+ rmContainer.getContainerId(), RMContainerEventType.ACQUIRED));
+ } else {
+ rmContainer.handle(new RMContainerUpdatesAcquiredEvent(
+ rmContainer.getContainerId(), increasedContainer));
}
+ return container;
}
- // Create container token and NMToken altogether, if either of them fails for
+ // Create container token and update NMToken altogether, if either of them fails for
// some reason like DNS unavailable, do not return this container and keep it
// in the newlyAllocatedContainers waiting to be refetched.
- public synchronized ContainersAndNMTokensAllocation
- pullNewlyAllocatedContainersAndNMTokens() {
+ public synchronized List<Container> pullNewlyAllocatedContainers() {
List<Container> returnContainerList =
new ArrayList<Container>(newlyAllocatedContainers.size());
- List<NMToken> nmTokens = new ArrayList<NMToken>();
for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
- .hasNext();) {
+ .hasNext();) {
RMContainer rmContainer = i.next();
- Container container = rmContainer.getContainer();
- ContainerType containerType = ContainerType.TASK;
- boolean isWaitingForAMContainer = isWaitingForAMContainer(
- container.getId().getApplicationAttemptId().getApplicationId());
- if (isWaitingForAMContainer) {
- containerType = ContainerType.APPLICATION_MASTER;
+ Container updatedContainer =
+ updateContainerAndNMToken(rmContainer, true, false);
+ // Only add container to return list when it's not null. updatedContainer
+ // could be null when generate token failed, it can be caused by DNS
+ // resolving failed.
+ if (updatedContainer != null) {
+ returnContainerList.add(updatedContainer);
+ i.remove();
}
- try {
- // create container token and NMToken altogether.
- container.setContainerToken(rmContext.getContainerTokenSecretManager()
- .createContainerToken(container.getId(), container.getNodeId(),
- getUser(), container.getResource(), container.getPriority(),
- rmContainer.getCreationTime(), this.logAggregationContext,
- rmContainer.getNodeLabelExpression(), containerType));
- NMToken nmToken =
- rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
- getApplicationAttemptId(), container);
- if (nmToken != null) {
- nmTokens.add(nmToken);
- }
- } catch (IllegalArgumentException e) {
- // DNS might be down, skip returning this container.
- LOG.error("Error trying to assign container token and NM token to" +
- " an allocated container " + container.getId(), e);
- continue;
+ }
+ return returnContainerList;
+ }
+
+ private synchronized List<Container> pullNewlyUpdatedContainers(
+ Map<ContainerId, RMContainer> updatedContainerMap, boolean increase) {
+ List<Container> returnContainerList =
+ new ArrayList<Container>(updatedContainerMap.size());
+ for (Iterator<Entry<ContainerId, RMContainer>> i =
+ updatedContainerMap.entrySet().iterator(); i.hasNext();) {
+ RMContainer rmContainer = i.next().getValue();
+ Container updatedContainer =
+ updateContainerAndNMToken(rmContainer, false, increase);
+ if (updatedContainer != null) {
+ returnContainerList.add(updatedContainer);
+ i.remove();
}
- returnContainerList.add(container);
- i.remove();
- rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
- RMContainerEventType.ACQUIRED));
}
- return new ContainersAndNMTokensAllocation(returnContainerList, nmTokens);
+ return returnContainerList;
+ }
+
+ public synchronized List<Container> pullNewlyIncreasedContainers() {
+ return pullNewlyUpdatedContainers(newlyIncreasedContainers, true);
+ }
+
+ public synchronized List<Container> pullNewlyDecreasedContainers() {
+ return pullNewlyUpdatedContainers(newlyDecreasedContainers, false);
+ }
+
+ public synchronized List<NMToken> pullUpdatedNMTokens() {
+ List<NMToken> returnList = new ArrayList<NMToken>(updatedNMTokens);
+ updatedNMTokens.clear();
+ return returnList;
}
public boolean isWaitingForAMContainer(ApplicationId applicationId) {
@@ -770,4 +835,50 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return attemptResourceUsage;
}
-}
+ public synchronized boolean removeIncreaseRequest(NodeId nodeId,
+ Priority priority, ContainerId containerId) {
+ return appSchedulingInfo.removeIncreaseRequest(nodeId, priority,
+ containerId);
+ }
+
+ public synchronized boolean updateIncreaseRequests(
+ List<SchedContainerChangeRequest> increaseRequests) {
+ return appSchedulingInfo.updateIncreaseRequests(increaseRequests);
+ }
+
+ private synchronized void changeContainerResource(
+ SchedContainerChangeRequest changeRequest, boolean increase) {
+ if (increase) {
+ appSchedulingInfo.increaseContainer(changeRequest);
+ } else {
+ appSchedulingInfo.decreaseContainer(changeRequest);
+ }
+
+ RMContainer changedRMContainer = changeRequest.getRMContainer();
+ changedRMContainer.handle(
+ new RMContainerChangeResourceEvent(changeRequest.getContainerId(),
+ changeRequest.getTargetCapacity(), increase));
+
+ // remove pending and not pulled by AM newly-increased/decreased-containers
+ // and add the new one
+ if (increase) {
+ newlyDecreasedContainers.remove(changeRequest.getContainerId());
+ newlyIncreasedContainers.put(changeRequest.getContainerId(),
+ changedRMContainer);
+ } else {
+ newlyIncreasedContainers.remove(changeRequest.getContainerId());
+ newlyDecreasedContainers.put(changeRequest.getContainerId(),
+ changedRMContainer);
+ }
+ }
+
+ public synchronized void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest) {
+ changeContainerResource(decreaseRequest, false);
+ }
+
+ public synchronized void increaseContainer(
+ SchedContainerChangeRequest increaseRequest) {
+ changeContainerResource(increaseRequest, true);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index f03663a..f3d3906 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -157,6 +157,37 @@ public abstract class SchedulerNode {
+ getUsedResource() + " used and " + getAvailableResource()
+ " available after allocation");
}
+
+ private synchronized void changeContainerResource(ContainerId containerId,
+ Resource deltaResource, boolean increase) {
+ if (increase) {
+ deductAvailableResource(deltaResource);
+ } else {
+ addAvailableResource(deltaResource);
+ }
+
+ LOG.info((increase ? "Increased" : "Decreased") + " container "
+ + containerId + " of capacity " + deltaResource + " on host "
+ + rmNode.getNodeAddress() + ", which has " + numContainers
+ + " containers, " + getUsedResource() + " used and "
+ + getAvailableResource() + " available after allocation");
+ }
+
+ /**
+ * The Scheduler increased container
+ */
+ public synchronized void increaseContainer(ContainerId containerId,
+ Resource deltaResource) {
+ changeContainerResource(containerId, deltaResource, true);
+ }
+
+ /**
+ * The Scheduler decreased container
+ */
+ public synchronized void decreaseContainer(ContainerId containerId,
+ Resource deltaResource) {
+ changeContainerResource(containerId, deltaResource, false);
+ }
/**
* Get available resources on the node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 8047d0b..abefee8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -361,7 +361,7 @@ public class SchedulerUtils {
}
public static boolean checkResourceRequestMatchingNodePartition(
- ResourceRequest offswitchResourceRequest, String nodePartition,
+ String requestedPartition, String nodePartition,
SchedulingMode schedulingMode) {
// We will only look at node label = nodeLabelToLookAt according to
// schedulingMode and partition of node.
@@ -371,12 +371,11 @@ public class SchedulerUtils {
} else {
nodePartitionToLookAt = RMNodeLabelsManager.NO_LABEL;
}
-
- String askedNodePartition = offswitchResourceRequest.getNodeLabelExpression();
- if (null == askedNodePartition) {
- askedNodePartition = RMNodeLabelsManager.NO_LABEL;
+
+ if (null == requestedPartition) {
+ requestedPartition = RMNodeLabelsManager.NO_LABEL;
}
- return askedNodePartition.equals(nodePartitionToLookAt);
+ return requestedPartition.equals(nodePartitionToLookAt);
}
private static boolean hasPendingResourceRequest(ResourceCalculator rc,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 699d476..0c2ae36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -133,16 +134,17 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
* @param release
* @param blacklistAdditions
* @param blacklistRemovals
+ * @param increaseRequests
+ * @param decreaseRequests
* @return the {@link Allocation} for the application
*/
@Public
@Stable
- Allocation
- allocate(ApplicationAttemptId appAttemptId,
- List<ResourceRequest> ask,
- List<ContainerId> release,
- List<String> blacklistAdditions,
- List<String> blacklistRemovals);
+ Allocation allocate(ApplicationAttemptId appAttemptId,
+ List<ResourceRequest> ask, List<ContainerId> release,
+ List<String> blacklistAdditions, List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests);
/**
* Get node resource usage report.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 0ae4d1a..9f61b11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -43,10 +43,10 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity;
import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -76,7 +76,7 @@ public abstract class AbstractCSQueue implements CSQueue {
private boolean preemptionDisabled;
// Track resource usage-by-label like used-resource/pending-resource, etc.
- ResourceUsage queueUsage;
+ volatile ResourceUsage queueUsage;
// Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
@@ -340,22 +340,27 @@ public abstract class AbstractCSQueue implements CSQueue {
return minimumAllocation;
}
- synchronized void allocateResource(Resource clusterResource,
- Resource resource, String nodePartition) {
+ synchronized void allocateResource(Resource clusterResource,
+ Resource resource, String nodePartition, boolean changeContainerResource) {
queueUsage.incUsed(nodePartition, resource);
- ++numContainers;
+ if (!changeContainerResource) {
+ ++numContainers;
+ }
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
minimumAllocation, this, labelManager, nodePartition);
}
protected synchronized void releaseResource(Resource clusterResource,
- Resource resource, String nodePartition) {
+ Resource resource, String nodePartition, boolean changeContainerResource) {
queueUsage.decUsed(nodePartition, resource);
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
minimumAllocation, this, labelManager, nodePartition);
- --numContainers;
+
+ if (!changeContainerResource) {
+ --numContainers;
+ }
}
@Private
@@ -446,8 +451,8 @@ public abstract class AbstractCSQueue implements CSQueue {
}
synchronized boolean canAssignToThisQueue(Resource clusterResource,
- String nodePartition, ResourceLimits currentResourceLimits, Resource resourceCouldBeUnreserved,
- SchedulingMode schedulingMode) {
+ String nodePartition, ResourceLimits currentResourceLimits,
+ Resource resourceCouldBeUnreserved, SchedulingMode schedulingMode) {
// Get current limited resource:
// - When doing RESPECT_PARTITION_EXCLUSIVITY allocation, we will respect
// queues' max capacity.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java
index 928437f..68f6f12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAssignment.java
@@ -41,6 +41,7 @@ public class CSAssignment {
private final boolean skipped;
private boolean fulfilledReservation;
private final AssignmentInformation assignmentInformation;
+ private boolean increaseAllocation;
public CSAssignment(Resource resource, NodeType type) {
this(resource, type, null, null, false, false);
@@ -138,4 +139,12 @@ public class CSAssignment {
public AssignmentInformation getAssignmentInformation() {
return this.assignmentInformation;
}
+
+ public boolean isIncreasedAllocation() {
+ return increaseAllocation;
+ }
+
+ public void setIncreasedAllocation(boolean flag) {
+ increaseAllocation = flag;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 9855dd4..e90deeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -219,6 +220,14 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
boolean sortQueues);
/**
+ * We have a reserved increased container in the queue, we need to unreserve
+ * it. Since we just want to cancel the reserved increase request instead of
+ * stop the container, we shouldn't call completedContainer for such purpose.
+ */
+ public void unreserveIncreasedContainer(Resource clusterResource,
+ FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer);
+
+ /**
* Get the number of applications in the queue.
* @return number of applications
*/
@@ -313,4 +322,11 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
* new resource asked
*/
public void decPendingResource(String nodeLabel, Resource resourceToDec);
+
+ /**
+ * Decrease container resource in the queue
+ */
+ public void decreaseContainer(Resource clusterResource,
+ SchedContainerChangeRequest decreaseRequest,
+ FiCaSchedulerApp app);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index ad5c76c..465e233 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
@@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -87,6 +89,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecreaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
@@ -98,6 +101,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerHealth;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -840,10 +845,14 @@ public class CapacityScheduler extends
}
@Override
+ // Note: when AM asks to decrease container or release container, we will
+ // acquire scheduler lock
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
- List<ResourceRequest> ask, List<ContainerId> release,
- List<String> blacklistAdditions, List<String> blacklistRemovals) {
+ List<ResourceRequest> ask, List<ContainerId> release,
+ List<String> blacklistAdditions, List<String> blacklistRemovals,
+ List<ContainerResourceChangeRequest> increaseRequests,
+ List<ContainerResourceChangeRequest> decreaseRequests) {
FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
if (application == null) {
@@ -854,6 +863,14 @@ public class CapacityScheduler extends
SchedulerUtils.normalizeRequests(
ask, getResourceCalculator(), getClusterResource(),
getMinimumResourceCapability(), getMaximumResourceCapability());
+
+ // Pre-process increase requests
+ List<SchedContainerChangeRequest> normalizedIncreaseRequests =
+ checkAndNormalizeContainerChangeRequests(increaseRequests, true);
+
+ // Pre-process decrease requests
+ List<SchedContainerChangeRequest> normalizedDecreaseRequests =
+ checkAndNormalizeContainerChangeRequests(decreaseRequests, false);
// Release containers
releaseContainers(release, application);
@@ -870,8 +887,8 @@ public class CapacityScheduler extends
return EMPTY_ALLOCATION;
}
+ // Process resource requests
if (!ask.isEmpty()) {
-
if(LOG.isDebugEnabled()) {
LOG.debug("allocate: pre-update " + applicationAttemptId +
" ask size =" + ask.size());
@@ -888,6 +905,12 @@ public class CapacityScheduler extends
application.showRequests();
}
}
+
+ // Process increase resource requests
+ if (application.updateIncreaseRequests(normalizedIncreaseRequests)
+ && (updateDemandForQueue == null)) {
+ updateDemandForQueue = (LeafQueue) application.getQueue();
+ }
if (application.isWaitingForAMContainer(application.getApplicationId())) {
// Allocate is for AM and update AM blacklist for this
@@ -896,6 +919,9 @@ public class CapacityScheduler extends
} else {
application.updateBlacklist(blacklistAdditions, blacklistRemovals);
}
+
+ // Decrease containers
+ decreaseContainers(normalizedDecreaseRequests, application);
allocation = application.getAllocation(getResourceCalculator(),
clusterResource, getMinimumResourceCapability());
@@ -957,6 +983,13 @@ public class CapacityScheduler extends
for (ContainerStatus launchedContainer : newlyLaunchedContainers) {
containerLaunchedOnNode(launchedContainer.getContainerId(), node);
}
+
+ // Processing the newly increased containers
+ List<Container> newlyIncreasedContainers =
+ nm.pullNewlyIncreasedContainers();
+ for (Container container : newlyIncreasedContainers) {
+ containerIncreasedOnNode(container.getId(), node, container);
+ }
// Process completed containers
int releasedContainers = 0;
@@ -1442,6 +1475,50 @@ public class CapacityScheduler extends
container.getId(), queue.getQueuePath());
}
}
+
+ @Lock(CapacityScheduler.class)
+ @Override
+ protected synchronized void decreaseContainer(
+ SchedContainerChangeRequest decreaseRequest,
+ SchedulerApplicationAttempt attempt) {
+ RMContainer rmContainer = decreaseRequest.getRMContainer();
+
+ // Check container status before doing decrease
+ if (rmContainer.getState() != RMContainerState.RUNNING) {
+ LOG.info("Trying to decrease a container not in RUNNING state, container="
+ + rmContainer + " state=" + rmContainer.getState().name());
+ return;
+ }
+
+ // Delta capacity of this decrease request is 0, this decrease request may
+ // just to cancel increase request
+ if (Resources.equals(decreaseRequest.getDeltaCapacity(), Resources.none())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Decrease target resource equals to existing resource for container:"
+ + decreaseRequest.getContainerId()
+ + " ignore this decrease request.");
+ }
+ return;
+ }
+
+ // Save resource before decrease
+ Resource resourceBeforeDecrease =
+ Resources.clone(rmContainer.getContainer().getResource());
+
+ FiCaSchedulerApp app = (FiCaSchedulerApp)attempt;
+ LeafQueue queue = (LeafQueue) attempt.getQueue();
+ queue.decreaseContainer(clusterResource, decreaseRequest, app);
+
+ // Notify RMNode the container will be decreased
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeDecreaseContainerEvent(decreaseRequest.getNodeId(),
+ Arrays.asList(rmContainer.getContainer())));
+
+ LOG.info("Application attempt " + app.getApplicationAttemptId()
+ + " decreased container:" + decreaseRequest.getContainerId() + " from "
+ + resourceBeforeDecrease + " to "
+ + decreaseRequest.getTargetCapacity());
+ }
@Lock(Lock.NoLock.class)
@VisibleForTesting
[28/32] hadoop git commit: YARN-1651. CapacityScheduler side changes
to support container resize. Contributed by Wangda Tan
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 85d2515..8fa1ad2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -18,44 +18,51 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import static java.lang.Thread.sleep;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
-import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.*;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.Assert;
-
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.lang.Thread.sleep;
public class TestApplicationMasterService {
private static final Log LOG = LogFactory
@@ -343,6 +350,92 @@ public class TestApplicationMasterService {
alloc1Response = am1.schedule();
Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size());
}
+
+ @Test(timeout=60000)
+ public void testInvalidIncreaseDecreaseRequest() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ MockRM rm = new MockRM(conf);
+
+ try {
+ rm.start();
+
+ // Register node1
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+
+ // Submit an application
+ RMApp app1 = rm.submitApp(1024);
+
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ RegisterApplicationMasterResponse registerResponse =
+ am1.registerAppAttempt();
+
+ sentRMContainerLaunched(rm,
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1));
+
+ // Ask for a normal increase should be successfull
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest.newInstance(
+ ContainerId.newContainerId(attempt1.getAppAttemptId(), 1),
+ Resources.createResource(2048))), null);
+
+ // Target resource is negative, should fail
+ boolean exceptionCaught = false;
+ try {
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest.newInstance(
+ ContainerId.newContainerId(attempt1.getAppAttemptId(), 1),
+ Resources.createResource(-1))), null);
+ } catch (InvalidResourceRequestException e) {
+ // This is expected
+ exceptionCaught = true;
+ }
+ Assert.assertTrue(exceptionCaught);
+
+ // Target resource is more than maxAllocation, should fail
+ try {
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest.newInstance(
+ ContainerId.newContainerId(attempt1.getAppAttemptId(), 1),
+ Resources
+ .add(registerResponse.getMaximumResourceCapability(),
+ Resources.createResource(1)))), null);
+ } catch (InvalidResourceRequestException e) {
+ // This is expected
+ exceptionCaught = true;
+ }
+
+ Assert.assertTrue(exceptionCaught);
+
+ // Contains multiple increase/decrease requests for same contaienrId
+ try {
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest.newInstance(
+ ContainerId.newContainerId(attempt1.getAppAttemptId(), 1),
+ Resources
+ .add(registerResponse.getMaximumResourceCapability(),
+ Resources.createResource(1)))), Arrays.asList(
+ ContainerResourceChangeRequest.newInstance(
+ ContainerId.newContainerId(attempt1.getAppAttemptId(), 1),
+ Resources
+ .add(registerResponse.getMaximumResourceCapability(),
+ Resources.createResource(1)))));
+ } catch (InvalidResourceRequestException e) {
+ // This is expected
+ exceptionCaught = true;
+ }
+
+ Assert.assertTrue(exceptionCaught);
+ } finally {
+ if (rm != null) {
+ rm.close();
+ }
+ }
+ }
private static class MyResourceManager extends MockRM {
@@ -354,4 +447,15 @@ public class TestApplicationMasterService {
return new DrainDispatcher();
}
}
+
+ private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ RMContainer rmContainer = cs.getRMContainer(containerId);
+ if (rmContainer != null) {
+ rmContainer.handle(
+ new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
+ } else {
+ Assert.fail("Cannot find RMContainer");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index dc843b9..168280a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
@@ -331,11 +332,15 @@ public class TestAMRestart {
MockAM am2 = MockRM.launchAM(app1, rm1, nm1);
RegisterApplicationMasterResponse registerResponse =
am2.registerAppAttempt();
- rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
+ rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// check am2 get the nm token from am1.
- Assert.assertEquals(expectedNMTokens,
- registerResponse.getNMTokensFromPreviousAttempts());
+ Assert.assertEquals(expectedNMTokens.size(),
+ registerResponse.getNMTokensFromPreviousAttempts().size());
+ for (int i = 0; i < expectedNMTokens.size(); i++) {
+ Assert.assertTrue(expectedNMTokens.get(i)
+ .equals(registerResponse.getNMTokensFromPreviousAttempts().get(i)));
+ }
// am2 allocate 1 container on nm2
containers = new ArrayList<Container>();
@@ -365,7 +370,7 @@ public class TestAMRestart {
// restart am
MockAM am3 = MockRM.launchAM(app1, rm1, nm1);
registerResponse = am3.registerAppAttempt();
- rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
+ rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// check am3 get the NM token from both am1 and am2;
List<NMToken> transferredTokens = registerResponse.getNMTokensFromPreviousAttempts();
@@ -430,7 +435,7 @@ public class TestAMRestart {
ContainerStatus containerStatus =
BuilderUtils.newContainerStatus(amContainer, ContainerState.COMPLETE,
- "", ContainerExitStatus.DISKS_FAILED);
+ "", ContainerExitStatus.DISKS_FAILED, Resources.createResource(200));
currentNode.containerStatus(containerStatus);
am1.waitForState(RMAppAttemptState.FAILED);
rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index 484a1b6..1f307aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -165,7 +165,7 @@ public class TestRMAppLogAggregationStatus {
node1ReportForApp.add(report1);
node1.handle(new RMNodeStatusEvent(node1.getNodeID(), NodeHealthStatus
.newInstance(true, null, 0), new ArrayList<ContainerStatus>(), null,
- null, node1ReportForApp));
+ null, node1ReportForApp, null));
List<LogAggregationReport> node2ReportForApp =
new ArrayList<LogAggregationReport>();
@@ -177,7 +177,7 @@ public class TestRMAppLogAggregationStatus {
node2ReportForApp.add(report2);
node2.handle(new RMNodeStatusEvent(node2.getNodeID(), NodeHealthStatus
.newInstance(true, null, 0), new ArrayList<ContainerStatus>(), null,
- null, node2ReportForApp));
+ null, node2ReportForApp, null));
// node1 and node2 has updated its log aggregation status
// verify that the log aggregation status for node1, node2
// has been changed
@@ -215,7 +215,7 @@ public class TestRMAppLogAggregationStatus {
node1ReportForApp2.add(report1_2);
node1.handle(new RMNodeStatusEvent(node1.getNodeID(), NodeHealthStatus
.newInstance(true, null, 0), new ArrayList<ContainerStatus>(), null,
- null, node1ReportForApp2));
+ null, node1ReportForApp2, null));
// verify that the log aggregation status for node1
// has been changed
@@ -284,7 +284,7 @@ public class TestRMAppLogAggregationStatus {
// 10 diagnostic messages/failure messages
node1.handle(new RMNodeStatusEvent(node1.getNodeID(), NodeHealthStatus
.newInstance(true, null, 0), new ArrayList<ContainerStatus>(), null,
- null, node1ReportForApp3));
+ null, node1ReportForApp3, null));
logAggregationStatus = rmApp.getLogAggregationReportsForApp();
Assert.assertEquals(2, logAggregationStatus.size());
@@ -329,7 +329,7 @@ public class TestRMAppLogAggregationStatus {
node2ReportForApp2.add(report2_3);
node2.handle(new RMNodeStatusEvent(node2.getNodeID(), NodeHealthStatus
.newInstance(true, null, 0), new ArrayList<ContainerStatus>(), null,
- null, node2ReportForApp2));
+ null, node2ReportForApp2, null));
Assert.assertEquals(LogAggregationStatus.FAILED,
rmApp.getLogAggregationStatusForAppReport());
logAggregationStatus = rmApp.getLogAggregationReportsForApp();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 10ec453..828e149 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -465,10 +465,9 @@ public class TestRMAppAttemptTransitions {
expectedAllocateCount = 1;
}
- assertEquals(expectedState,
- applicationAttempt.getAppAttemptState());
- verify(scheduler, times(expectedAllocateCount)).
- allocate(any(ApplicationAttemptId.class),
+ assertEquals(expectedState, applicationAttempt.getAppAttemptState());
+ verify(scheduler, times(expectedAllocateCount)).allocate(
+ any(ApplicationAttemptId.class), any(List.class), any(List.class),
any(List.class), any(List.class), any(List.class), any(List.class));
assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
@@ -488,11 +487,9 @@ public class TestRMAppAttemptTransitions {
assertEquals(amContainer, applicationAttempt.getMasterContainer());
// Check events
verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class));
- verify(scheduler, times(2)).
- allocate(
- any(
- ApplicationAttemptId.class), any(List.class), any(List.class),
- any(List.class), any(List.class));
+ verify(scheduler, times(2)).allocate(any(ApplicationAttemptId.class),
+ any(List.class), any(List.class), any(List.class), any(List.class),
+ any(List.class), any(List.class));
verify(nmTokenManager).clearNodeSetForAttempt(
applicationAttempt.getAppAttemptId());
}
@@ -641,13 +638,9 @@ public class TestRMAppAttemptTransitions {
Allocation allocation = mock(Allocation.class);
when(allocation.getContainers()).
thenReturn(Collections.singletonList(container));
- when(
- scheduler.allocate(
- any(ApplicationAttemptId.class),
- any(List.class),
- any(List.class),
- any(List.class),
- any(List.class))).
+ when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class),
+ any(List.class), any(List.class), any(List.class), any(List.class),
+ any(List.class))).
thenReturn(allocation);
RMContainer rmContainer = mock(RMContainerImpl.class);
when(scheduler.getRMContainer(container.getId())).
@@ -1511,10 +1504,9 @@ public class TestRMAppAttemptTransitions {
@Test
public void testScheduleTransitionReplaceAMContainerRequestWithDefaults() {
YarnScheduler mockScheduler = mock(YarnScheduler.class);
- when(
- mockScheduler.allocate(any(ApplicationAttemptId.class),
- any(List.class), any(List.class), any(List.class), any(List.class)))
- .thenAnswer(new Answer<Allocation>() {
+ when(mockScheduler.allocate(any(ApplicationAttemptId.class),
+ any(List.class), any(List.class), any(List.class), any(List.class),
+ any(List.class), any(List.class))).thenAnswer(new Answer<Allocation>() {
@SuppressWarnings("rawtypes")
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index e4e2049..415e891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
-import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
@@ -191,6 +190,10 @@ public class TestRMContainerImpl {
Container container = BuilderUtils.newContainer(containerId, nodeId,
"host:3465", resource, priority, null);
+
+ ConcurrentMap<ApplicationId, RMApp> appMap = new ConcurrentHashMap<>();
+ RMApp rmApp = mock(RMApp.class);
+ appMap.putIfAbsent(appId, rmApp);
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
@@ -200,6 +203,7 @@ public class TestRMContainerImpl {
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
+ when(rmContext.getRMApps()).thenReturn(appMap);
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@@ -235,11 +239,118 @@ public class TestRMContainerImpl {
rmContainer.handle(new RMContainerFinishedEvent(containerId,
containerStatus, RMContainerEventType.EXPIRE));
drainDispatcher.await();
+ assertEquals(RMContainerState.EXPIRED, rmContainer.getState());
+ verify(writer, times(1)).containerFinished(any(RMContainer.class));
+ verify(publisher, times(1)).containerFinished(any(RMContainer.class),
+ anyLong());
+ }
+
+ private void testExpireAfterIncreased(boolean acquired) {
+ /*
+ * Similar to previous test, a container is increased but not acquired by
+ * AM. In this case, if a container is expired, the container should be
+ * finished.
+ */
+ DrainDispatcher drainDispatcher = new DrainDispatcher();
+ EventHandler<RMAppAttemptEvent> appAttemptEventHandler =
+ mock(EventHandler.class);
+ EventHandler generic = mock(EventHandler.class);
+ drainDispatcher.register(RMAppAttemptEventType.class,
+ appAttemptEventHandler);
+ drainDispatcher.register(RMNodeEventType.class, generic);
+ drainDispatcher.init(new YarnConfiguration());
+ drainDispatcher.start();
+ NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 1);
+ ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
+ ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
+
+ Resource resource = BuilderUtils.newResource(512, 1);
+ Priority priority = BuilderUtils.newPriority(5);
+
+ Container container = BuilderUtils.newContainer(containerId, nodeId,
+ "host:3465", resource, priority, null);
+
+ RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
+ SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
+ when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
+ when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
+ when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
+ ConcurrentMap<ApplicationId, RMApp> apps =
+ new ConcurrentHashMap<ApplicationId, RMApp>();
+ apps.put(appId, mock(RMApp.class));
+ when(rmContext.getRMApps()).thenReturn(apps);
+ RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
+ nodeId, "user", rmContext);
+
+ assertEquals(RMContainerState.NEW, rmContainer.getState());
+ assertEquals(resource, rmContainer.getAllocatedResource());
+ assertEquals(nodeId, rmContainer.getAllocatedNode());
+ assertEquals(priority, rmContainer.getAllocatedPriority());
+ verify(writer).containerStarted(any(RMContainer.class));
+ verify(publisher).containerCreated(any(RMContainer.class), anyLong());
+
+ rmContainer.handle(new RMContainerEvent(containerId,
+ RMContainerEventType.START));
+ drainDispatcher.await();
+ assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
+
+ rmContainer.handle(new RMContainerEvent(containerId,
+ RMContainerEventType.ACQUIRED));
+ drainDispatcher.await();
+ assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
+
+ rmContainer.handle(new RMContainerEvent(containerId,
+ RMContainerEventType.LAUNCHED));
+ drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
- verify(writer, never()).containerFinished(any(RMContainer.class));
- verify(publisher, never()).containerFinished(any(RMContainer.class),
+ assertEquals(
+ "http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ rmContainer.getLogURL());
+
+ // newResource is more than the old resource
+ Resource newResource = BuilderUtils.newResource(1024, 2);
+ rmContainer.handle(new RMContainerChangeResourceEvent(containerId,
+ newResource, true));
+
+ if (acquired) {
+ rmContainer
+ .handle(new RMContainerUpdatesAcquiredEvent(containerId, true));
+ drainDispatcher.await();
+ // status is still RUNNING since this is a increased container acquired by
+ // AM
+ assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+ }
+
+ // In RUNNING state. Verify EXPIRE and associated actions.
+ reset(appAttemptEventHandler);
+ ContainerStatus containerStatus = SchedulerUtils
+ .createAbnormalContainerStatus(containerId,
+ SchedulerUtils.EXPIRED_CONTAINER);
+ rmContainer.handle(new RMContainerFinishedEvent(containerId,
+ containerStatus, RMContainerEventType.EXPIRE));
+ drainDispatcher.await();
+ assertEquals(RMContainerState.EXPIRED, rmContainer.getState());
+
+ // Container will be finished only when it is acquired by AM after increase,
+ // we will only notify expirer when it is acquired by AM.
+ verify(writer, times(1)).containerFinished(any(RMContainer.class));
+ verify(publisher, times(1)).containerFinished(any(RMContainer.class),
anyLong());
}
+
+ @Test
+ public void testExpireAfterContainerResourceIncreased() throws Exception {
+ // expire after increased and acquired by AM
+ testExpireAfterIncreased(true);
+ // expire after increased but not acquired by AM
+ testExpireAfterIncreased(false);
+ }
@Test
public void testExistenceOfResourceRequestInRMContainer() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 88c1444..7f6a749 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -31,7 +31,6 @@ import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -59,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -103,6 +103,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptI
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -139,7 +141,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.Mockito;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -678,11 +679,11 @@ public class TestCapacityScheduler {
// Verify the blacklist can be updated independent of requesting containers
cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(),
- Collections.singletonList(host), null);
+ Collections.singletonList(host), null, null, null);
Assert.assertTrue(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
- Collections.singletonList(host));
+ Collections.singletonList(host), null, null);
Assert.assertFalse(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
rm.stop();
}
@@ -777,7 +778,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId1,
Collections.<ResourceRequest>singletonList(r1),
Collections.<ContainerId>emptyList(),
- null, null);
+ null, null, null, null);
//And this will result in container assignment for app1
CapacityScheduler.schedule(cs);
@@ -794,7 +795,7 @@ public class TestCapacityScheduler {
cs.allocate(appAttemptId2,
Collections.<ResourceRequest>singletonList(r2),
Collections.<ContainerId>emptyList(),
- null, null);
+ null, null, null, null);
//In this case we do not perform container assignment because we want to
//verify re-ordering based on the allocation alone
@@ -2907,7 +2908,7 @@ public class TestCapacityScheduler {
Allocation allocate =
cs.allocate(appAttemptId, Collections.<ResourceRequest> emptyList(),
- Collections.<ContainerId> emptyList(), null, null);
+ Collections.<ContainerId> emptyList(), null, null, null, null);
Assert.assertNotNull(attempt);
@@ -2923,7 +2924,7 @@ public class TestCapacityScheduler {
allocate =
cs.allocate(appAttemptId, Collections.<ResourceRequest> emptyList(),
- Collections.<ContainerId> emptyList(), null, null);
+ Collections.<ContainerId> emptyList(), null, null, null, null);
// All resources should be sent as headroom
Assert.assertEquals(newResource, allocate.getResourceLimit());
@@ -3084,7 +3085,107 @@ public class TestCapacityScheduler {
config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class.getName());
verifyAMLimitForLeafQueue(config);
+ }
+
+ private FiCaSchedulerApp getFiCaSchedulerApp(MockRM rm,
+ ApplicationId appId) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ return cs.getSchedulerApplications().get(appId).getCurrentAppAttempt();
+ }
+ @Test
+ public void testPendingResourceUpdatedAccordingToIncreaseRequestChanges()
+ throws Exception {
+ Configuration conf =
+ TestUtils.getConfigurationWithQueueLabels(new Configuration(false));
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+
+ final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+ mgr.init(conf);
+
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ MockRM rm = new MockRM(conf, memStore) {
+ protected RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm.start();
+
+ MockNM nm1 = // label = ""
+ new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ // Launch app1 in queue=a1
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ // Allocate two more containers
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(2 * GB), 2)),
+ null);
+ ContainerId containerId1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ ContainerId containerId3 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
+ Assert.assertTrue(rm.waitForState(nm1, containerId3,
+ RMContainerState.ALLOCATED, 10 * 1000));
+ // Acquire them
+ am1.allocate(null, null);
+ sentRMContainerLaunched(rm,
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1L));
+ sentRMContainerLaunched(rm,
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L));
+ sentRMContainerLaunched(rm,
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 3L));
+
+ // am1 asks to change its AM container from 1GB to 3GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(3 * GB))),
+ null);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm, app1.getApplicationId());
+
+ Assert.assertEquals(2 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ checkPendingResource(rm, "a1", 2 * GB, null);
+ checkPendingResource(rm, "a", 2 * GB, null);
+ checkPendingResource(rm, "root", 2 * GB, null);
+
+ // am1 asks to change containerId2 (2G -> 3G) and containerId3 (2G -> 5G)
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId2, Resources.createResource(3 * GB)),
+ ContainerResourceChangeRequest
+ .newInstance(containerId3, Resources.createResource(5 * GB))),
+ null);
+
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ checkPendingResource(rm, "a1", 6 * GB, null);
+ checkPendingResource(rm, "a", 6 * GB, null);
+ checkPendingResource(rm, "root", 6 * GB, null);
+
+ // am1 asks to change containerId1 (1G->3G), containerId2 (2G -> 4G) and
+ // containerId3 (2G -> 2G)
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(3 * GB)),
+ ContainerResourceChangeRequest
+ .newInstance(containerId2, Resources.createResource(4 * GB)),
+ ContainerResourceChangeRequest
+ .newInstance(containerId3, Resources.createResource(2 * GB))),
+ null);
+ Assert.assertEquals(4 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ checkPendingResource(rm, "a1", 4 * GB, null);
+ checkPendingResource(rm, "a", 4 * GB, null);
+ checkPendingResource(rm, "root", 4 * GB, null);
}
private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config)
@@ -3146,4 +3247,15 @@ public class TestCapacityScheduler {
+ CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
conf.setInt(propName, maxAllocVcores);
}
+
+ private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ RMContainer rmContainer = cs.getRMContainer(containerId);
+ if (rmContainer != null) {
+ rmContainer.handle(
+ new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
+ } else {
+ Assert.fail("Cannot find RMContainer");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 9dcab2e..88c7c13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -132,11 +132,11 @@ public class TestChildQueueOrder {
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
((ParentQueue)queue).allocateResource(clusterResource,
- allocatedResource, RMNodeLabelsManager.NO_LABEL);
+ allocatedResource, RMNodeLabelsManager.NO_LABEL, false);
} else {
FiCaSchedulerApp app1 = getMockApplication(0, "");
((LeafQueue)queue).allocateResource(clusterResource, app1,
- allocatedResource, null, null);
+ allocatedResource, null, null, false);
}
// Next call - nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 769041b..b5b2222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -60,9 +59,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
public class TestContainerAllocation {
@@ -199,13 +195,16 @@ public class TestContainerAllocation {
// acquire the container.
SecurityUtilTestHelper.setTokenServiceUseIp(true);
- List<Container> containers =
- am1.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
- // not able to fetch the container;
- Assert.assertEquals(0, containers.size());
-
- SecurityUtilTestHelper.setTokenServiceUseIp(false);
+ List<Container> containers;
+ try {
+ containers =
+ am1.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ // not able to fetch the container;
+ Assert.assertEquals(0, containers.size());
+ } finally {
+ SecurityUtilTestHelper.setTokenServiceUseIp(false);
+ }
containers =
am1.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
@@ -315,21 +314,24 @@ public class TestContainerAllocation {
rm1.start();
MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
- SecurityUtilTestHelper.setTokenServiceUseIp(true);
- RMApp app1 = rm1.submitApp(200);
- RMAppAttempt attempt = app1.getCurrentAppAttempt();
- nm1.nodeHeartbeat(true);
-
- // fetching am container will fail, keep retrying 5 times.
- while (numRetries <= 5) {
+ RMApp app1;
+ try {
+ SecurityUtilTestHelper.setTokenServiceUseIp(true);
+ app1 = rm1.submitApp(200);
+ RMAppAttempt attempt = app1.getCurrentAppAttempt();
nm1.nodeHeartbeat(true);
- Thread.sleep(1000);
- Assert.assertEquals(RMAppAttemptState.SCHEDULED,
- attempt.getAppAttemptState());
- System.out.println("Waiting for am container to be allocated.");
- }
- SecurityUtilTestHelper.setTokenServiceUseIp(false);
+ // fetching am container will fail, keep retrying 5 times.
+ while (numRetries <= 5) {
+ nm1.nodeHeartbeat(true);
+ Thread.sleep(1000);
+ Assert.assertEquals(RMAppAttemptState.SCHEDULED,
+ attempt.getAppAttemptState());
+ System.out.println("Waiting for am container to be allocated.");
+ }
+ } finally {
+ SecurityUtilTestHelper.setTokenServiceUseIp(false);
+ }
MockRM.launchAndRegisterAM(app1, rm1, nm1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
new file mode 100644
index 0000000..23283f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -0,0 +1,963 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerResourceChangeRequest;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestContainerResizing {
+ private final int GB = 1024;
+
+ private YarnConfiguration conf;
+
+ RMNodeLabelsManager mgr;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ mgr = new NullRMNodeLabelsManager();
+ mgr.init(conf);
+ }
+
+ @Test
+ public void testSimpleIncreaseContainer() throws Exception {
+ /**
+ * Application has a container running, and the node has enough available
+ * resource. Add a increase request to see if container will be increased
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ ContainerId containerId1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ sentRMContainerLaunched(rm1, containerId1);
+ // am1 asks to change its AM container from 1GB to 3GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(3 * GB))),
+ null);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ checkPendingResource(rm1, "default", 2 * GB, null);
+ Assert.assertEquals(2 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // NM1 do 1 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // Pending resource should be deducted
+ checkPendingResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ verifyContainerIncreased(am1.allocate(null, null), containerId1, 3 * GB);
+ verifyAvailableResourceOfSchedulerNode(rm1, nm1.getNodeId(), 17 * GB);
+
+ rm1.close();
+ }
+
+ @Test
+ public void testSimpleDecreaseContainer() throws Exception {
+ /**
+ * Application has a container running, try to decrease the container and
+ * check queue's usage and container resource will be updated.
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(3 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ checkUsedResource(rm1, "default", 3 * GB, null);
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ ContainerId containerId1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ sentRMContainerLaunched(rm1, containerId1);
+
+ // am1 asks to change its AM container from 1GB to 3GB
+ AllocateResponse response = am1.sendContainerResizingRequest(null, Arrays
+ .asList(ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(1 * GB))));
+
+ verifyContainerDecreased(response, containerId1, 1 * GB);
+ checkUsedResource(rm1, "default", 1 * GB, null);
+ Assert.assertEquals(1 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ // Check if decreased containers added to RMNode
+ RMNodeImpl rmNode =
+ (RMNodeImpl) rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ Collection<Container> decreasedContainers =
+ rmNode.getToBeDecreasedContainers();
+ boolean rmNodeReceivedDecreaseContainer = false;
+ for (Container c : decreasedContainers) {
+ if (c.getId().equals(containerId1)
+ && c.getResource().equals(Resources.createResource(1 * GB))) {
+ rmNodeReceivedDecreaseContainer = true;
+ }
+ }
+ Assert.assertTrue(rmNodeReceivedDecreaseContainer);
+
+ rm1.close();
+ }
+
+ @Test
+ public void testSimpleIncreaseRequestReservation() throws Exception {
+ /**
+ * Application has two containers running, try to increase one of then, node
+ * doesn't have enough resource, so the increase request will be reserved.
+ * Check resource usage after container reserved, finish a container, the
+ * reserved container should be allocated.
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ // Allocate two more containers
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(2 * GB), 1)),
+ null);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(rm1.waitForState(nm1, containerId2,
+ RMContainerState.ALLOCATED, 10 * 1000));
+ // Acquire them, and NM report RUNNING
+ am1.allocate(null, null);
+ sentRMContainerLaunched(rm1, containerId2);
+
+ ContainerId containerId1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ sentRMContainerLaunched(rm1, containerId1);
+
+
+ // am1 asks to change its AM container from 1GB to 3GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(7 * GB))),
+ null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // NM1 do 1 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId1);
+
+ /* Check reservation statuses */
+ // Increase request should be reserved
+ Assert.assertTrue(rmContainer1.hasIncreaseReservation());
+ Assert.assertEquals(6 * GB, rmContainer1.getReservedResource().getMemory());
+ Assert.assertFalse(app.getReservedContainers().isEmpty());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Pending resource will not be changed since it's not satisfied
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 9 * GB, null);
+ Assert.assertEquals(9 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+
+ // Complete one container and do another allocation
+ am1.allocate(null, Arrays.asList(containerId2));
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // Now container should be increased
+ verifyContainerIncreased(am1.allocate(null, null), containerId1, 7 * GB);
+
+ /* Check statuses after reservation satisfied */
+ // Increase request should be unreserved
+ Assert.assertFalse(rmContainer1.hasIncreaseReservation());
+ Assert.assertTrue(app.getReservedContainers().isEmpty());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Pending resource will be changed since it's satisfied
+ checkPendingResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 7 * GB, null);
+ Assert.assertEquals(7 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(7 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+ verifyAvailableResourceOfSchedulerNode(rm1, nm1.getNodeId(), 1 * GB);
+
+ rm1.close();
+ }
+
+ @Test
+ public void testExcessiveReservationWhenCancelIncreaseRequest()
+ throws Exception {
+ /**
+ * Application has two containers running, try to increase one of then, node
+ * doesn't have enough resource, so the increase request will be reserved.
+ * Check resource usage after container reserved, finish a container &
+ * cancel the increase request, reservation should be cancelled
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ // Allocate two more containers
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(2 * GB), 1)),
+ null);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(rm1.waitForState(nm1, containerId2,
+ RMContainerState.ALLOCATED, 10 * 1000));
+ // Acquire them, and NM report RUNNING
+ am1.allocate(null, null);
+ sentRMContainerLaunched(rm1, containerId2);
+
+ ContainerId containerId1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ sentRMContainerLaunched(rm1, containerId1);
+
+ // am1 asks to change its AM container from 1GB to 3GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(7 * GB))),
+ null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // NM1 do 1 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId1);
+
+ /* Check reservation statuses */
+ // Increase request should be reserved
+ Assert.assertTrue(rmContainer1.hasIncreaseReservation());
+ Assert.assertEquals(6 * GB, rmContainer1.getReservedResource().getMemory());
+ Assert.assertFalse(app.getReservedContainers().isEmpty());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Pending resource will not be changed since it's not satisfied
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 9 * GB, null);
+ Assert.assertEquals(9 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+
+ // Complete one container and cancel increase request (via send a increase
+ // request, make target_capacity=existing_capacity)
+ am1.allocate(null, Arrays.asList(containerId2));
+ // am1 asks to change its AM container from 1G to 1G (cancel the increase
+ // request actually)
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(1 * GB))),
+ null);
+ // Trigger a node heartbeat..
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ /* Check statuses after reservation satisfied */
+ // Increase request should be unreserved
+ Assert.assertTrue(app.getReservedContainers().isEmpty());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ Assert.assertFalse(rmContainer1.hasIncreaseReservation());
+ // Pending resource will be changed since it's satisfied
+ checkPendingResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 1 * GB, null);
+ Assert.assertEquals(1 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(1 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ rm1.close();
+ }
+
+ @Test
+ public void testExcessiveReservationWhenDecreaseSameContainer()
+ throws Exception {
+ /**
+ * Very similar to testExcessiveReservationWhenCancelIncreaseRequest, after
+ * the increase request reserved, it decreases the reserved container,
+ * container should be decreased and reservation will be cancelled
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ // Allocate two more containers
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(2 * GB), 1)),
+ null);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(rm1.waitForState(nm1, containerId2,
+ RMContainerState.ALLOCATED, 10 * 1000));
+ // Acquire them, and NM report RUNNING
+ am1.allocate(null, null);
+ sentRMContainerLaunched(rm1, containerId2);
+
+ ContainerId containerId1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+ sentRMContainerLaunched(rm1, containerId1);
+
+
+ // am1 asks to change its AM container from 2GB to 8GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(8 * GB))),
+ null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // NM1 do 1 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ RMContainer rmContainer1 = app.getLiveContainersMap().get(containerId1);
+
+ /* Check reservation statuses */
+ // Increase request should be reserved
+ Assert.assertTrue(rmContainer1.hasIncreaseReservation());
+ Assert.assertEquals(6 * GB, rmContainer1.getReservedResource().getMemory());
+ Assert.assertFalse(app.getReservedContainers().isEmpty());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Pending resource will not be changed since it's not satisfied
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 10 * GB, null);
+ Assert.assertEquals(10 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(4 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+
+ // Complete one container and cancel increase request (via send a increase
+ // request, make target_capacity=existing_capacity)
+ am1.allocate(null, Arrays.asList(containerId2));
+ // am1 asks to change its AM container from 2G to 1G (decrease)
+ am1.sendContainerResizingRequest(null, Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId1, Resources.createResource(1 * GB))));
+ // Trigger a node heartbeat..
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ /* Check statuses after reservation satisfied */
+ // Increase request should be unreserved
+ Assert.assertTrue(app.getReservedContainers().isEmpty());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ Assert.assertFalse(rmContainer1.hasIncreaseReservation());
+ // Pending resource will be changed since it's satisfied
+ checkPendingResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 1 * GB, null);
+ Assert.assertEquals(1 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(1 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ rm1.close();
+ }
+
+ @Test
+ public void testIncreaseContainerUnreservedWhenContainerCompleted()
+ throws Exception {
+ /**
+ * App has two containers on the same node (node.resource = 8G), container1
+ * = 2G, container2 = 2G. App asks to increase container2 to 8G.
+ *
+ * So increase container request will be reserved. When app releases
+ * container2, reserved part should be released as well.
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ // Allocate two more containers
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(2 * GB), 1)),
+ null);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(rm1.waitForState(nm1, containerId2,
+ RMContainerState.ALLOCATED, 10 * 1000));
+ // Acquire them, and NM report RUNNING
+ am1.allocate(null, null);
+ sentRMContainerLaunched(rm1, containerId2);
+ rm1.waitForContainerState(containerId2, RMContainerState.RUNNING);
+
+ // am1 asks to change its AM container from 2GB to 8GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId2, Resources.createResource(8 * GB))),
+ null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // NM1 do 1 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ RMContainer rmContainer2 = app.getLiveContainersMap().get(containerId2);
+
+ /* Check reservation statuses */
+ // Increase request should be reserved
+ Assert.assertTrue(rmContainer2.hasIncreaseReservation());
+ Assert.assertEquals(6 * GB, rmContainer2.getReservedResource().getMemory());
+ Assert.assertFalse(app.getReservedContainers().isEmpty());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Pending resource will not be changed since it's not satisfied
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 9 * GB, null);
+ Assert.assertEquals(9 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+
+ // Complete container2, container will be unreserved and completed
+ am1.allocate(null, Arrays.asList(containerId2));
+
+ /* Check statuses after reservation satisfied */
+ // Increase request should be unreserved
+ Assert.assertTrue(app.getReservedContainers().isEmpty());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ Assert.assertFalse(rmContainer2.hasIncreaseReservation());
+ // Pending resource will be changed since it's satisfied
+ checkPendingResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 1 * GB, null);
+ Assert.assertEquals(1 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(1 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ rm1.close();
+ }
+
+ @Test
+ public void testIncreaseContainerUnreservedWhenApplicationCompleted()
+ throws Exception {
+ /**
+ * Similar to testIncreaseContainerUnreservedWhenContainerCompleted, when
+ * application finishes, reserved increase container should be cancelled
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+
+ // Allocate two more containers
+ am1.allocate(
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(2 * GB), 1)),
+ null);
+ ContainerId containerId2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(
+ rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED,
+ 10 * 1000));
+ // Acquire them, and NM report RUNNING
+ am1.allocate(null, null);
+ sentRMContainerLaunched(rm1, containerId2);
+
+ // am1 asks to change its AM container from 2GB to 8GB
+ am1.sendContainerResizingRequest(Arrays.asList(
+ ContainerResourceChangeRequest
+ .newInstance(containerId2, Resources.createResource(8 * GB))),
+ null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // NM1 do 1 heartbeats
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ RMContainer rmContainer2 = app.getLiveContainersMap().get(containerId2);
+
+ /* Check reservation statuses */
+ // Increase request should be reserved
+ Assert.assertTrue(rmContainer2.hasIncreaseReservation());
+ Assert.assertEquals(6 * GB, rmContainer2.getReservedResource().getMemory());
+ Assert.assertFalse(app.getReservedContainers().isEmpty());
+ Assert.assertNotNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ // Pending resource will not be changed since it's not satisfied
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 9 * GB, null);
+ Assert.assertEquals(9 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+
+ // Kill the application
+ cs.handle(new AppAttemptRemovedSchedulerEvent(am1.getApplicationAttemptId(),
+ RMAppAttemptState.KILLED, false));
+
+ /* Check statuses after reservation satisfied */
+ // Increase request should be unreserved
+ Assert.assertTrue(app.getReservedContainers().isEmpty());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()).getReservedContainer());
+ Assert.assertFalse(rmContainer2.hasIncreaseReservation());
+ // Pending resource will be changed since it's satisfied
+ checkPendingResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 0 * GB, null);
+ Assert.assertEquals(0 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ rm1.close();
+ }
+
+ private void allocateAndLaunchContainers(MockAM am, MockNM nm, MockRM rm,
+ int nContainer, int mem, int priority, int startContainerId)
+ throws Exception {
+ am.allocate(Arrays
+ .asList(ResourceRequest.newInstance(Priority.newInstance(priority), "*",
+ Resources.createResource(mem), nContainer)),
+ null);
+ ContainerId lastContainerId = ContainerId.newContainerId(
+ am.getApplicationAttemptId(), startContainerId + nContainer - 1);
+ Assert.assertTrue(rm.waitForState(nm, lastContainerId,
+ RMContainerState.ALLOCATED, 10 * 1000));
+ // Acquire them, and NM report RUNNING
+ am.allocate(null, null);
+
+ for (int cId = startContainerId; cId < startContainerId
+ + nContainer; cId++) {
+ sentRMContainerLaunched(rm,
+ ContainerId.newContainerId(am.getApplicationAttemptId(), cId));
+ rm.waitForContainerState(
+ ContainerId.newContainerId(am.getApplicationAttemptId(), cId),
+ RMContainerState.RUNNING);
+ }
+ }
+
+ @Test
+ public void testOrderOfIncreaseContainerRequestAllocation()
+ throws Exception {
+ /**
+ * There're multiple containers need to be increased, check container will
+ * be increase sorted by priority, if priority is same, smaller containerId
+ * container will get preferred
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+ ApplicationAttemptId attemptId = am1.getApplicationAttemptId();
+
+ // Container 2, 3 (priority=3)
+ allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 3, 2);
+
+ // Container 4, 5 (priority=2)
+ allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 2, 4);
+
+ // Container 6, 7 (priority=4)
+ allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 4, 6);
+
+ // am1 asks to change its container[2-7] from 1G to 2G
+ List<ContainerResourceChangeRequest> increaseRequests = new ArrayList<>();
+ for (int cId = 2; cId <= 7; cId++) {
+ ContainerId containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), cId);
+ increaseRequests.add(ContainerResourceChangeRequest
+ .newInstance(containerId, Resources.createResource(2 * GB)));
+ }
+ am1.sendContainerResizingRequest(increaseRequests, null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // Get rmNode1
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ // assignContainer, container-4/5/2 increased (which has highest priority OR
+ // earlier allocated)
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ AllocateResponse allocateResponse = am1.allocate(null, null);
+ Assert.assertEquals(3, allocateResponse.getIncreasedContainers().size());
+ verifyContainerIncreased(allocateResponse,
+ ContainerId.newContainerId(attemptId, 4), 2 * GB);
+ verifyContainerIncreased(allocateResponse,
+ ContainerId.newContainerId(attemptId, 5), 2 * GB);
+ verifyContainerIncreased(allocateResponse,
+ ContainerId.newContainerId(attemptId, 2), 2 * GB);
+
+ /* Check statuses after allocation */
+ // There're still 3 pending increase requests
+ checkPendingResource(rm1, "default", 3 * GB, null);
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 10 * GB, null);
+ Assert.assertEquals(10 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(10 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ rm1.close();
+ }
+
+ @Test
+ public void testIncreaseContainerRequestGetPreferrence()
+ throws Exception {
+ /**
+ * There're multiple containers need to be increased, and there're several
+ * container allocation request, scheduler will try to increase container
+ * before allocate new containers
+ */
+ MockRM rm1 = new MockRM() {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
+
+ // app1 -> a1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ FiCaSchedulerApp app = getFiCaSchedulerApp(rm1, app1.getApplicationId());
+ ApplicationAttemptId attemptId = am1.getApplicationAttemptId();
+
+ // Container 2, 3 (priority=3)
+ allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 3, 2);
+
+ // Container 4, 5 (priority=2)
+ allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 2, 4);
+
+ // Container 6, 7 (priority=4)
+ allocateAndLaunchContainers(am1, nm1, rm1, 2, 1 * GB, 4, 6);
+
+ // am1 asks to change its container[2-7] from 1G to 2G
+ List<ContainerResourceChangeRequest> increaseRequests = new ArrayList<>();
+ for (int cId = 2; cId <= 7; cId++) {
+ ContainerId containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), cId);
+ increaseRequests.add(ContainerResourceChangeRequest
+ .newInstance(containerId, Resources.createResource(2 * GB)));
+ }
+ am1.sendContainerResizingRequest(increaseRequests, null);
+
+ checkPendingResource(rm1, "default", 6 * GB, null);
+ Assert.assertEquals(6 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+
+ // Get rmNode1
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ // assignContainer, container-4/5/2 increased (which has highest priority OR
+ // earlier allocated)
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ AllocateResponse allocateResponse = am1.allocate(null, null);
+ Assert.assertEquals(3, allocateResponse.getIncreasedContainers().size());
+ verifyContainerIncreased(allocateResponse,
+ ContainerId.newContainerId(attemptId, 4), 2 * GB);
+ verifyContainerIncreased(allocateResponse,
+ ContainerId.newContainerId(attemptId, 5), 2 * GB);
+ verifyContainerIncreased(allocateResponse,
+ ContainerId.newContainerId(attemptId, 2), 2 * GB);
+
+ /* Check statuses after allocation */
+ // There're still 3 pending increase requests
+ checkPendingResource(rm1, "default", 3 * GB, null);
+ Assert.assertEquals(3 * GB,
+ app.getAppAttemptResourceUsage().getPending().getMemory());
+ // Queue/user/application's usage will be updated
+ checkUsedResource(rm1, "default", 10 * GB, null);
+ Assert.assertEquals(10 * GB, ((LeafQueue) cs.getQueue("default"))
+ .getUser("user").getUsed().getMemory());
+ Assert.assertEquals(0 * GB,
+ app.getAppAttemptResourceUsage().getReserved().getMemory());
+ Assert.assertEquals(10 * GB,
+ app.getAppAttemptResourceUsage().getUsed().getMemory());
+
+ rm1.close();
+ }
+
+ private void checkPendingResource(MockRM rm, String queueName, int memory,
+ String label) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ CSQueue queue = cs.getQueue(queueName);
+ Assert.assertEquals(memory,
+ queue.getQueueResourceUsage()
+ .getPending(label == null ? RMNodeLabelsManager.NO_LABEL : label)
+ .getMemory());
+ }
+
+ private void checkUsedResource(MockRM rm, String queueName, int memory,
+ String label) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ CSQueue queue = cs.getQueue(queueName);
+ Assert.assertEquals(memory,
+ queue.getQueueResourceUsage()
+ .getUsed(label == null ? RMNodeLabelsManager.NO_LABEL : label)
+ .getMemory());
+ }
+
+ private void verifyContainerIncreased(AllocateResponse response,
+ ContainerId containerId, int mem) {
+ List<Container> increasedContainers = response.getIncreasedContainers();
+ boolean found = false;
+ for (Container c : increasedContainers) {
+ if (c.getId().equals(containerId)) {
+ found = true;
+ Assert.assertEquals(mem, c.getResource().getMemory());
+ }
+ }
+ if (!found) {
+ Assert.fail("Container not increased: containerId=" + containerId);
+ }
+ }
+
+ private void verifyContainerDecreased(AllocateResponse response,
+ ContainerId containerId, int mem) {
+ List<Container> decreasedContainers = response.getDecreasedContainers();
+ boolean found = false;
+ for (Container c : decreasedContainers) {
+ if (c.getId().equals(containerId)) {
+ found = true;
+ Assert.assertEquals(mem, c.getResource().getMemory());
+ }
+ }
+ if (!found) {
+ Assert.fail("Container not decreased: containerId=" + containerId);
+ }
+ }
+
+ private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ RMContainer rmContainer = cs.getRMContainer(containerId);
+ if (rmContainer != null) {
+ rmContainer.handle(
+ new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
+ } else {
+ Assert.fail("Cannot find RMContainer");
+ }
+ }
+
+ private void verifyAvailableResourceOfSchedulerNode(MockRM rm, NodeId nodeId,
+ int expectedMemory) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ SchedulerNode node = cs.getNode(nodeId);
+ Assert
+ .assertEquals(expectedMemory, node.getAvailableResource().getMemory());
+ }
+
+ private FiCaSchedulerApp getFiCaSchedulerApp(MockRM rm,
+ ApplicationId appId) {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ return cs.getSchedulerApplications().get(appId).getCurrentAppAttempt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874cc98e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index fe8be06..b85c697 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -770,9 +770,9 @@ public class TestLeafQueue {
qb.finishApplication(app_0.getApplicationId(), user_0);
qb.finishApplication(app_2.getApplicationId(), user_1);
qb.releaseResource(clusterResource, app_0, app_0.getResource(u0Priority),
- null, null);
+ null, null, false);
qb.releaseResource(clusterResource, app_2, app_2.getResource(u1Priority),
- null, null);
+ null, null, false);
qb.setUserLimit(50);
qb.setUserLimitFactor(1);
[05/32] hadoop git commit: MAPREDUCE-6460.
TestRMContainerAllocator.testAttemptNotFoundCausesRMCommunicatorException
fails. Contributed by Zhihai Xu.
Posted by wa...@apache.org.
MAPREDUCE-6460. TestRMContainerAllocator.testAttemptNotFoundCausesRMCommunicatorException fails. Contributed by Zhihai Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3ace31e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3ace31e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3ace31e
Branch: refs/heads/YARN-1197
Commit: e3ace31e2b7e3a8dc6cf3dc044d5ef2a9aa12e29
Parents: 66b46d0
Author: Zhihai Xu <zx...@apache.org>
Authored: Sat Sep 19 00:10:06 2015 -0700
Committer: Zhihai Xu <zx...@apache.org>
Committed: Sat Sep 19 00:10:06 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3ace31e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index c67e494..42ea011 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -568,6 +568,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5002. AM could potentially allocate a reduce container to a map
attempt (Chang Li via jlowe)
+ MAPREDUCE-6460. TestRMContainerAllocator.
+ testAttemptNotFoundCausesRMCommunicatorException fails. (Zhihai Xu)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3ace31e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index c98ccd3..1a3829e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -125,6 +125,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -2809,7 +2810,7 @@ public class TestRMContainerAllocator {
// Now kill the application
rm.killApp(app.getApplicationId());
-
+ rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
allocator.schedule();
}
[03/32] hadoop git commit: YARN-3920. FairScheduler container
reservation on a node should be configurable to limit it to large containers
(adhoot via asuresh)
Posted by wa...@apache.org.
YARN-3920. FairScheduler container reservation on a node should be configurable to limit it to large containers (adhoot via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94dec5a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94dec5a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94dec5a9
Branch: refs/heads/YARN-1197
Commit: 94dec5a9164cd9bc573fbf74e76bcff9e7c5c637
Parents: 602335d
Author: Arun Suresh <as...@apache.org>
Authored: Fri Sep 18 14:00:49 2015 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Sep 18 14:02:55 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/fair/FSAppAttempt.java | 19 ++++-
.../scheduler/fair/FairScheduler.java | 21 ++++-
.../fair/FairSchedulerConfiguration.java | 22 ++++-
.../scheduler/fair/FairSchedulerTestBase.java | 6 ++
.../scheduler/fair/TestFairScheduler.java | 89 +++++++++++++++++++-
6 files changed, 148 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dec5a9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 822624f..7487f71 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -454,6 +454,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6478. Add an option to skip cleanupJob stage or ignore cleanup
failure during commitJob. (Junping Du via wangda)
+ YARN-3920. FairScheduler container reservation on a node should be
+ configurable to limit it to large containers (adhoot via asuresh)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dec5a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index cfec915..7af1891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -543,10 +543,23 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return container.getResource();
}
- // The desired container won't fit here, so reserve
- reserve(request.getPriority(), node, container, reserved);
+ if (isReservable(container)) {
+ // The desired container won't fit here, so reserve
+ reserve(request.getPriority(), node, container, reserved);
- return FairScheduler.CONTAINER_RESERVED;
+ return FairScheduler.CONTAINER_RESERVED;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not creating reservation as container " + container.getId()
+ + " is not reservable");
+ }
+ return Resources.none();
+ }
+ }
+
+ private boolean isReservable(Container container) {
+ return scheduler.isAtLeastReservationThreshold(
+ getQueue().getPolicy().getResourceCalculator(), container.getResource());
}
private boolean hasNodeOrRackLocalRequests(Priority priority) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dec5a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 3a39799..a083272 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -194,7 +194,11 @@ public class FairScheduler extends
private AllocationFileLoaderService allocsLoader;
@VisibleForTesting
AllocationConfiguration allocConf;
-
+
+ // Container size threshold for making a reservation.
+ @VisibleForTesting
+ Resource reservationThreshold;
+
public FairScheduler() {
super(FairScheduler.class.getName());
clock = new SystemClock();
@@ -203,6 +207,12 @@ public class FairScheduler extends
maxRunningEnforcer = new MaxRunningAppsEnforcer(this);
}
+ public boolean isAtLeastReservationThreshold(
+ ResourceCalculator resourceCalculator, Resource resource) {
+ return Resources.greaterThanOrEqual(
+ resourceCalculator, clusterResource, resource, reservationThreshold);
+ }
+
private void validateConf(Configuration conf) {
// validate scheduler memory allocation setting
int minMem = conf.getInt(
@@ -1325,6 +1335,7 @@ public class FairScheduler extends
minimumAllocation = this.conf.getMinimumAllocation();
initMaximumResourceCapability(this.conf.getMaximumAllocation());
incrAllocation = this.conf.getIncrementAllocation();
+ updateReservationThreshold();
continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled();
continuousSchedulingSleepMs =
this.conf.getContinuousSchedulingSleepMs();
@@ -1391,6 +1402,14 @@ public class FairScheduler extends
}
}
+ private void updateReservationThreshold() {
+ Resource newThreshold = Resources.multiply(
+ getIncrementResourceCapability(),
+ this.conf.getReservationThresholdIncrementMultiple());
+
+ reservationThreshold = newThreshold;
+ }
+
private synchronized void startSchedulerThreads() {
Preconditions.checkNotNull(updateThread, "updateThread is null");
Preconditions.checkNotNull(allocsLoader, "allocsLoader is null");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dec5a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index e477e6e..892484d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -49,7 +47,17 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES =
YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores";
public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1;
-
+
+ /** Threshold for container size for making a container reservation as a
+ * multiple of increment allocation. Only container sizes above this are
+ * allowed to reserve a node */
+ public static final String
+ RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE =
+ YarnConfiguration.YARN_PREFIX +
+ "scheduler.reservation-threshold.increment-multiple";
+ public static final float
+ DEFAULT_RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE = 2f;
+
private static final String CONF_PREFIX = "yarn.scheduler.fair.";
public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
@@ -166,7 +174,13 @@ public class FairSchedulerConfiguration extends Configuration {
DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES);
return Resources.createResource(incrementMemory, incrementCores);
}
-
+
+ public float getReservationThresholdIncrementMultiple() {
+ return getFloat(
+ RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ DEFAULT_RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE);
+ }
+
public float getLocalityThresholdNode() {
return getFloat(LOCALITY_THRESHOLD_NODE, DEFAULT_LOCALITY_THRESHOLD_NODE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dec5a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 1c9801d..dd7ed41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -64,6 +64,7 @@ public class FairSchedulerTestBase {
protected Configuration conf;
protected FairScheduler scheduler;
protected ResourceManager resourceManager;
+ public static final float TEST_RESERVATION_THRESHOLD = 0.09f;
// Helper methods
public Configuration createConfiguration() {
@@ -76,6 +77,11 @@ public class FairSchedulerTestBase {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false);
conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
+
+ conf.setFloat(
+ FairSchedulerConfiguration
+ .RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ TEST_RESERVATION_THRESHOLD);
return conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94dec5a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index a02cf18..ad54616 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -710,9 +710,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
// Asked for less than increment allocation.
- assertEquals(FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ assertEquals(
+ FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(updateEvent2);
@@ -764,7 +765,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemory());
+ getResourceUsage().getMemory());
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemory());
// Now another node checks in with capacity
@@ -939,8 +940,88 @@ public class TestFairScheduler extends FairSchedulerTestBase {
getResourceUsage().getMemory());
}
-
+ @Test
+ public void testReservationThresholdGatesReservations() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<defaultQueueSchedulingPolicy>drf" +
+ "</defaultQueueSchedulingPolicy>");
+ out.println("</allocations>");
+ out.close();
+ // Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will
+ // use vcores)
+ conf.setFloat(FairSchedulerConfiguration.
+ RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ 2f);
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node
+ RMNode node1 =
+ MockNodes
+ .newNodeInfo(1, Resources.createResource(4096, 4), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue 1 requests full capacity of node
+ createSchedulingRequest(4096, 4, "queue1", "user1", 1, 1);
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+
+ scheduler.handle(updateEvent);
+
+ // Make sure queue 1 is allocated app capacity
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getResourceUsage().getMemory());
+
+ // Now queue 2 requests below threshold
+ ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Make sure queue 2 has no reservation
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getResourceUsage().getMemory());
+ assertEquals(0,
+ scheduler.getSchedulerApp(attId).getReservedContainers().size());
+
+ // Now queue requests CPU above threshold
+ createSchedulingRequestExistingApplication(1024, 3, 1, attId);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Make sure queue 2 is waiting with a reservation
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getResourceUsage().getMemory());
+ assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
+ .getVirtualCores());
+
+ // Now another node checks in with capacity
+ RMNode node2 =
+ MockNodes
+ .newNodeInfo(1, Resources.createResource(1024, 4), 2, "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+ scheduler.handle(updateEvent2);
+
+ // Make sure this goes to queue 2
+ assertEquals(3, scheduler.getQueueManager().getQueue("queue2").
+ getResourceUsage().getVirtualCores());
+
+ // The old reservation should still be there...
+ assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
+ .getVirtualCores());
+ // ... but it should disappear when we update the first node.
+ scheduler.handle(updateEvent);
+ assertEquals(0, scheduler.getSchedulerApp(attId).getCurrentReservation()
+ .getVirtualCores());
+ }
@Test
public void testEmptyQueueName() throws Exception {