You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/10/12 09:37:28 UTC
[01/50] [abbrv] hadoop git commit: HDFS-12606. When using native
decoder,
DFSStripedStream.close crashes JVM after being called multiple times. (Lei
(Eddy) Xu) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5881 95a819343 -> 8cc00b7f7 (forced update)
HDFS-12606. When using native decoder, DFSStripedStream.close crashes JVM after being called multiple times. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46644319
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46644319
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46644319
Branch: refs/heads/YARN-5881
Commit: 46644319e1b3295ddbc7597c060956bf46487d11
Parents: 6d6ca4c
Author: Lei Xu <le...@apache.org>
Authored: Mon Oct 9 10:08:30 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Mon Oct 9 10:08:30 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSStripedInputStream.java | 7 +++++--
.../apache/hadoop/hdfs/TestDFSStripedInputStream.java | 13 +++++++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46644319/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index d4d0646..e7d90ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -68,7 +68,7 @@ public class DFSStripedInputStream extends DFSInputStream {
private ByteBuffer curStripeBuf;
private ByteBuffer parityBuf;
private final ErasureCodingPolicy ecPolicy;
- private final RawErasureDecoder decoder;
+ private RawErasureDecoder decoder;
/**
* Indicate the start/end offset of the current buffered stripe in the
@@ -188,7 +188,10 @@ public class DFSStripedInputStream extends DFSInputStream {
BUFFER_POOL.putBuffer(parityBuf);
parityBuf = null;
}
- decoder.release();
+ if (decoder != null) {
+ decoder.release();
+ decoder = null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46644319/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index f94b7ab..de276a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -491,4 +491,17 @@ public class TestDFSStripedInputStream {
assertEquals(readSize, done);
assertArrayEquals(expected, readBuffer);
}
+
+ @Test
+ public void testIdempotentClose() throws Exception {
+ final int numBlocks = 2;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
+ stripesPerBlock, false, ecPolicy);
+
+ try (DFSInputStream in = fs.getClient().open(filePath.toString())) {
+ assertTrue(in instanceof DFSStripedInputStream);
+ // Close twice
+ in.close();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-7309.
TestClientRMService#testUpdateApplicationPriorityRequest and
TestClientRMService#testUpdatePriorityAndKillAppWithZeroClusterResource test
functionality not supported by FairScheduler (rkanter)
Posted by su...@apache.org.
YARN-7309. TestClientRMService#testUpdateApplicationPriorityRequest and TestClientRMService#testUpdatePriorityAndKillAppWithZeroClusterResource test functionality not supported by FairScheduler (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec8bf9e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec8bf9e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec8bf9e4
Branch: refs/heads/YARN-5881
Commit: ec8bf9e48ac4dc675516a57c06a29890b5138548
Parents: 132cdac
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Oct 10 10:14:42 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Oct 10 10:18:42 2017 -0700
----------------------------------------------------------------------
.../yarn/server/resourcemanager/TestClientRMService.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec8bf9e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 35b3f86..ae57dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -148,6 +148,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
@@ -159,6 +160,7 @@ import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
@@ -1945,6 +1947,9 @@ public class TestClientRMService {
int maxPriority = 10;
int appPriority = 5;
YarnConfiguration conf = new YarnConfiguration();
+ Assume.assumeFalse("FairScheduler does not support Application Priorities",
+ conf.get(YarnConfiguration.RM_SCHEDULER)
+ .equals(FairScheduler.class.getName()));
conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY,
maxPriority);
MockRM rm = new MockRM(conf);
@@ -1963,6 +1968,9 @@ public class TestClientRMService {
int maxPriority = 10;
int appPriority = 5;
YarnConfiguration conf = new YarnConfiguration();
+ Assume.assumeFalse("FairScheduler does not support Application Priorities",
+ conf.get(YarnConfiguration.RM_SCHEDULER)
+ .equals(FairScheduler.class.getName()));
conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY,
maxPriority);
MockRM rm = new MockRM(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDFS-12603. Enable async edit
logging by default.
Posted by su...@apache.org.
HDFS-12603. Enable async edit logging by default.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afb42aea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afb42aea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afb42aea
Branch: refs/heads/YARN-5881
Commit: afb42aeabf1317b755ab79e0265bc90920c896ac
Parents: 09ad848
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Oct 9 11:20:00 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Oct 9 11:20:00 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 7 ++++++-
.../hdfs/server/namenode/FSEditLogAsync.java | 5 +++++
.../src/main/resources/hdfs-default.xml | 2 +-
.../namenode/ha/TestFailureToReadEdits.java | 22 +++++++++++++-------
5 files changed, 28 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afb42aea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1f96763..395b192 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
- public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+ public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afb42aea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index a8f5bfa..7d19b04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
*
* In a non-HA setup:
*
- * The log starts in UNITIALIZED state upon construction. Once it's
+ * The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
@@ -1837,4 +1837,9 @@ public class FSEditLog implements LogsPurgeable {
}
return count;
}
+
+ @Override
+ public String toString() {
+ return super.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afb42aea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
index c14a310..1f5dc75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
@@ -319,4 +319,9 @@ class FSEditLogAsync extends FSEditLog implements Runnable {
return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]";
}
}
+
+ @Override
+ public String toString() {
+ return super.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afb42aea/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3491ed2..3752578 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4105,7 +4105,7 @@
<property>
<name>dfs.namenode.edits.asynclogging</name>
- <value>false</value>
+ <value>true</value>
<description>
If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afb42aea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index 93c717c..2bc246d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random();
private final TestType clusterType;
+ private final boolean useAsyncEditLogging;
private Configuration conf;
private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@@ -88,18 +89,23 @@ public class TestFailureToReadEdits {
};
/**
- * Run this suite of tests both for QJM-based HA and for file-based
- * HA.
+ * Run this suite of tests for {QJM-based, file-based HA} x {async
+ * edit logging enabled, disabled}.
*/
@Parameters
public static Iterable<Object[]> data() {
- return Arrays.asList(new Object[][] {
- { TestType.SHARED_DIR_HA },
- { TestType.QJM_HA } });
+ return Arrays.asList(new Object[][]{
+ {TestType.SHARED_DIR_HA, Boolean.FALSE},
+ {TestType.SHARED_DIR_HA, Boolean.TRUE},
+ {TestType.QJM_HA, Boolean.FALSE},
+ {TestType.QJM_HA, Boolean.TRUE},
+ });
}
-
- public TestFailureToReadEdits(TestType clusterType) {
+
+ public TestFailureToReadEdits(TestType clusterType, Boolean
+ useAsyncEditLogging) {
this.clusterType = clusterType;
+ this.useAsyncEditLogging = useAsyncEditLogging;
}
@Before
@@ -109,6 +115,8 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-5953 addendum: Move
QueueConfigInfo and SchedConfUpdateInfo to package
org.apache.hadoop.yarn.webapp.dao
Posted by su...@apache.org.
YARN-5953 addendum: Move QueueConfigInfo and SchedConfUpdateInfo to package org.apache.hadoop.yarn.webapp.dao
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04f9e80b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04f9e80b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04f9e80b
Branch: refs/heads/YARN-5881
Commit: 04f9e80bb2662c51c2fc53c2edc10c660fd71a9e
Parents: 916bdbd
Author: Xuan <xg...@apache.org>
Authored: Mon Jul 31 11:49:05 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 57 +++++++++++++
.../yarn/webapp/dao/SchedConfUpdateInfo.java | 85 ++++++++++++++++++++
.../webapp/dao/QueueConfigInfo.java | 57 -------------
.../webapp/dao/SchedConfUpdateInfo.java | 85 --------------------
4 files changed, 142 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04f9e80b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
new file mode 100644
index 0000000..d1d91c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.webapp.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for adding or updating a queue to scheduler configuration
+ * for this queue.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigInfo {
+
+ @XmlElement(name = "queueName")
+ private String queue;
+
+ private HashMap<String, String> params = new HashMap<>();
+
+ public QueueConfigInfo() { }
+
+ public QueueConfigInfo(String queue, Map<String, String> params) {
+ this.queue = queue;
+ this.params = new HashMap<>(params);
+ }
+
+ public String getQueue() {
+ return this.queue;
+ }
+
+ public HashMap<String, String> getParams() {
+ return this.params;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04f9e80b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
new file mode 100644
index 0000000..bb84096
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue, as well as global scheduler conf changes).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SchedConfUpdateInfo {
+
+ @XmlElement(name = "add-queue")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove-queue")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update-queue")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ private HashMap<String, String> global = new HashMap<>();
+
+ public SchedConfUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public void setAddQueueInfo(ArrayList<QueueConfigInfo> addQueueInfo) {
+ this.addQueueInfo = addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public void setRemoveQueueInfo(ArrayList<String> removeQueueInfo) {
+ this.removeQueueInfo = removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+
+ public void setUpdateQueueInfo(ArrayList<QueueConfigInfo> updateQueueInfo) {
+ this.updateQueueInfo = updateQueueInfo;
+ }
+
+ @XmlElementWrapper(name = "global-updates")
+ public HashMap<String, String> getGlobalParams() {
+ return global;
+ }
+
+ public void setGlobalParams(HashMap<String, String> globalInfo) {
+ this.global = globalInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04f9e80b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
deleted file mode 100644
index d1d91c2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.webapp.dao;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for adding or updating a queue to scheduler configuration
- * for this queue.
- */
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class QueueConfigInfo {
-
- @XmlElement(name = "queueName")
- private String queue;
-
- private HashMap<String, String> params = new HashMap<>();
-
- public QueueConfigInfo() { }
-
- public QueueConfigInfo(String queue, Map<String, String> params) {
- this.queue = queue;
- this.params = new HashMap<>(params);
- }
-
- public String getQueue() {
- return this.queue;
- }
-
- public HashMap<String, String> getParams() {
- return this.params;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04f9e80b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
deleted file mode 100644
index bb84096..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.webapp.dao;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlElementWrapper;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for making scheduler configuration changes (supports adding,
- * removing, or updating a queue, as well as global scheduler conf changes).
- */
-@XmlRootElement(name = "schedConf")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class SchedConfUpdateInfo {
-
- @XmlElement(name = "add-queue")
- private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "remove-queue")
- private ArrayList<String> removeQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "update-queue")
- private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
-
- private HashMap<String, String> global = new HashMap<>();
-
- public SchedConfUpdateInfo() {
- // JAXB needs this
- }
-
- public ArrayList<QueueConfigInfo> getAddQueueInfo() {
- return addQueueInfo;
- }
-
- public void setAddQueueInfo(ArrayList<QueueConfigInfo> addQueueInfo) {
- this.addQueueInfo = addQueueInfo;
- }
-
- public ArrayList<String> getRemoveQueueInfo() {
- return removeQueueInfo;
- }
-
- public void setRemoveQueueInfo(ArrayList<String> removeQueueInfo) {
- this.removeQueueInfo = removeQueueInfo;
- }
-
- public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
- return updateQueueInfo;
- }
-
- public void setUpdateQueueInfo(ArrayList<QueueConfigInfo> updateQueueInfo) {
- this.updateQueueInfo = updateQueueInfo;
- }
-
- @XmlElementWrapper(name = "global-updates")
- public HashMap<String, String> getGlobalParams() {
- return global;
- }
-
- public void setGlobalParams(HashMap<String, String> globalInfo) {
- this.global = globalInfo;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-7238. Documentation for API
based scheduler configuration management
Posted by su...@apache.org.
YARN-7238. Documentation for API based scheduler configuration management
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3075de9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3075de9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3075de9
Branch: refs/heads/YARN-5881
Commit: a3075de9ab2fc9a97debaab6fc838ea07b1cc77e
Parents: d6622da
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Sep 25 17:47:32 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../src/site/markdown/CapacityScheduler.md | 24 ++-
.../src/site/markdown/ResourceManagerRest.md | 186 +++++++++++++++++++
.../src/site/markdown/YarnCommands.md | 13 ++
3 files changed, 221 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3075de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 6bb8489..d70f891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -295,9 +295,29 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach
Changing Queue Configuration
----------------------------
-Changing queue properties and adding new queues is very simple. You need to edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*.
+Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API.
+
+### Changing queue configuration via file
+
+ To edit by file, you need to edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*.
$ vi $HADOOP_CONF_DIR/capacity-scheduler.xml
$ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues
-**Note:** Queues cannot be *deleted*, only addition of new queues is supported - the updated queue configuration should be a valid one i.e. queue-capacity at each *level* should be equal to 100%.
+### Changing queue configuration via API
+
+ Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml.
+
+ | Property | Description |
+ |:---- |:---- |
+ | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. |
+ | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). |
+ | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. |
+ | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. |
+ | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. |
+ | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. |
+ | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. |
+
+ **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
+
+ See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3075de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index 82f80f3..025e9a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -4430,3 +4430,189 @@ Response Body:
<remainingTimeInSeconds>90</remainingTimeInSeconds>
</timeout>
```
+
+Scheduler Configuration Mutation API
+--------------------------------
+
+The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy.
+
+
+### URI
+
+ * http://rm-http-address:port/ws/v1/cluster/sched-conf
+
+### HTTP Operations Supported
+
+ * PUT
+
+### Elements of the *schedConf* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| update-queue | object | A queue whose configurations should be updated |
+| add-queue | object | A queue to add to the scheduler along with this queue's configurations |
+| remove-queue | string | Full path name of a queue to remove |
+| global-updates | map | Map of key value pairs to update scheduler's global configuration |
+
+### PUT Request Examples
+
+Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response.
+
+**Updating queue configuration(s)**
+
+Request for updating queue configurations.
+
+*Elements of the* update-queue *object*
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| queueName | string | Full path name of the queue to update |
+| params | map | A map of key value configuration pairs to update for this queue |
+
+Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10.
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <update-queue>
+ <queueName>root.default</queueName>
+ <params>
+ <entry>
+ <key>maximum-applications</key>
+ <value>100</value>
+ </entry>
+ <entry>
+ <key>minimum-user-limit-percent</key>
+ <value>10</value>
+ </entry>
+ </params>
+ </update-queue>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
+
+
+**Adding a queue**
+
+Request for adding queues/updating queue configurations.
+
+*Elements of the* add-queue *object*
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| queueName | string | Full path name of the queue to add |
+| params | map | A map of key value configuration pairs to set for this queue |
+
+Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.)
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <add-queue>
+ <queueName>root.a</queueName>
+ <params>
+ <entry>
+ <key>capacity</key>
+ <value>10</value>
+ </entry>
+ <entry>
+ <key>maximum-capacity</key>
+ <value>10</value>
+ </entry>
+ </params>
+ </add-queue>
+ <update-queue>
+ <queueName>root.default</queueName>
+ <params>
+ <entry>
+ <key>capacity</key>
+ <value>90</value>
+ </entry>
+ <entry>
+ <key>maximum-capacity</key>
+ <value>90</value>
+ </entry>
+ </params>
+ </update-queue>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
+
+**Removing queues**
+
+Request for removing queues from the queue hierarchy.
+
+Assuming we are using the capacity scheduler and the current queue configuration is three queues *root.default*, *root.a*, and *root.b*, this example removes both *root.a* and *root.b*. (More complex examples include removing a parent queue and its children.)
+
+**Note:** Queues must be put into `STOPPED` state before they are deleted. Any updated queue configuration should be a valid one i.e. queue-capacity at each *level* should be equal to 100%.
+
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <remove-queue>root.a</remove-queue>
+ <remove-queue>root.b</remove-queue>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
+
+**Updating global scheduler configurations**
+
+Request for updating global scheduler configurations. Assuming we are using the capacity scheduler, this example enables queue mappings. For global configuration updates, the full configuration key must be specified.
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <global-updates>
+ <entry>
+ <key>yarn.scheduler.capacity.queue-mappings-override.enable</key>
+ <value>true</value>
+ </entry>
+ </global-updates>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3075de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 5f430ec..2fc65bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -237,6 +237,19 @@ Usage:
Runs ResourceManager admin client
+### schedconf
+
+Usage: `yarn schedconf [options]`
+
+| COMMAND\_OPTIONS | Description |
+|:---- |:---- |
+| -add <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. |
+| -remove <queuePath1,queuePath2> | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. |
+| -update <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. |
+| -global <key1=val1,key2=val2> | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. |
+
+Updates scheduler configuration
+
### scmadmin
Usage: `yarn scmadmin [options] `
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-5949. Add pluggable
configuration ACL policy interface and implementation. (Jonathan Hung via
wangda)
Posted by su...@apache.org.
YARN-5949. Add pluggable configuration ACL policy interface and implementation. (Jonathan Hung via wangda)
Change-Id: Ib98e82ff753bede21fcab2e6ca9ec1e7a5a2008f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4e62530
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4e62530
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4e62530
Branch: refs/heads/YARN-5881
Commit: a4e62530469e4c3d5b339a06adeac2146fc15fa5
Parents: d8a5136
Author: Wangda Tan <wa...@apache.org>
Authored: Mon May 22 13:38:31 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../src/main/resources/yarn-default.xml | 11 ++
.../ConfigurationMutationACLPolicy.java | 47 ++++++
.../ConfigurationMutationACLPolicyFactory.java | 49 ++++++
.../DefaultConfigurationMutationACLPolicy.java | 45 ++++++
.../scheduler/MutableConfScheduler.java | 19 ++-
.../scheduler/MutableConfigurationProvider.java | 8 +-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 151 +++++++++++++++++-
...ueueAdminConfigurationMutationACLPolicy.java | 96 ++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 131 +---------------
.../TestConfigurationMutationACLPolicies.java | 154 +++++++++++++++++++
.../TestMutableCSConfigurationProvider.java | 40 +++--
13 files changed, 610 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 92b9697..b603135 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -680,6 +680,9 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
+ public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
+ YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 73f4717..e124c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3379,4 +3379,15 @@
<value>memory</value>
</property>
+ <property>
+ <description>
+ The class to use for configuration mutation ACL policy if using a mutable
+ configuration provider. Controls whether a mutation request is allowed.
+ The DefaultConfigurationMutationACLPolicy checks if the requestor is a
+ YARN admin.
+ </description>
+ <name>yarn.scheduler.configuration.mutation.acl-policy.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..724487b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Interface for determining whether configuration mutations are allowed.
+ */
+public interface ConfigurationMutationACLPolicy {
+
+ /**
+ * Initialize ACL policy with configuration and RMContext.
+ * @param conf Configuration to initialize with.
+ * @param rmContext rmContext
+ */
+ void init(Configuration conf, RMContext rmContext);
+
+ /**
+ * Check if mutation is allowed.
+ * @param user User issuing the request
+ * @param confUpdate configurations to be updated
+ * @return whether provided mutation is allowed or not
+ */
+ boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
new file mode 100644
index 0000000..2898785
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of
+ * {@link ConfigurationMutationACLPolicy}.
+ */
+public final class ConfigurationMutationACLPolicyFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ ConfigurationMutationACLPolicyFactory.class);
+
+ private ConfigurationMutationACLPolicyFactory() {
+ // Unused.
+ }
+
+ public static ConfigurationMutationACLPolicy getPolicy(Configuration conf) {
+ Class<? extends ConfigurationMutationACLPolicy> policyClass =
+ conf.getClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ LOG.info("Using ConfigurationMutationACLPolicy implementation - " +
+ policyClass);
+ return ReflectionUtils.newInstance(policyClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..680c3b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Default configuration mutation ACL policy. Checks if user is YARN admin.
+ */
+public class DefaultConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private YarnAuthorizationProvider authorizer;
+
+ @Override
+ public void init(Configuration conf, RMContext rmContext) {
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ return authorizer.isAdmin(user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 35e36e1..93a935e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -17,10 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
-import java.util.Map;
/**
* Interface for a scheduler that supports changing configuration at runtime.
@@ -31,10 +32,22 @@ public interface MutableConfScheduler extends ResourceScheduler {
/**
* Update the scheduler's configuration.
* @param user Caller of this update
- * @param confUpdate key-value map of the configuration update
+ * @param confUpdate configuration update
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException;
+ QueueConfigsUpdateInfo confUpdate) throws IOException;
+ /**
+ * Get the scheduler configuration.
+ * @return the scheduler configuration
+ */
+ Configuration getConfiguration();
+
+ /**
+ * Get queue object based on queue name.
+ * @param queueName the queue name
+ * @return the queue object
+ */
+ Queue getQueue(String queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 889c3bc..f04c128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
import java.io.IOException;
-import java.util.Map;
/**
* Interface for allowing changing scheduler configurations.
@@ -32,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate)
- throws IOException;
+ void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 7c5839b..4f89f7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -649,6 +650,7 @@ public class CapacityScheduler extends
preemptionManager.refreshQueues(null, this.getRootQueue());
}
+ @Override
public CSQueue getQueue(String queueName) {
if (queueName == null) {
return null;
@@ -2615,10 +2617,10 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException {
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
- user.getShortUserName(), confUpdate);
+ user, confUpdate);
} else {
throw new UnsupportedOperationException("Configured CS configuration " +
"provider does not support updating configuration.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ea1b3c0..8b879b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -18,14 +18,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.base.Joiner;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -38,6 +51,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Configuration schedConf;
private YarnConfigurationStore confStore;
+ private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
private Configuration conf;
@@ -68,6 +82,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
confStore.initialize(config, schedConf);
+ this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
+ .getPolicy(config);
+ aclMutationPolicy.init(config, rmContext);
this.conf = config;
}
@@ -80,12 +97,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public void mutateConfiguration(String user,
- Map<String, String> confUpdate) throws IOException {
+ public void mutateConfiguration(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
+ if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
+ throw new AccessControlException("User is not admin of all modified" +
+ " queues.");
+ }
Configuration oldConf = new Configuration(schedConf);
- LogMutation log = new LogMutation(confUpdate, user);
+ Map<String, String> kvUpdate = constructKeyValueConfUpdate(confUpdate);
+ LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
long id = confStore.logMutation(log);
- for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ for (Map.Entry<String, String> kv : kvUpdate.entrySet()) {
if (kv.getValue() == null) {
schedConf.unset(kv.getKey());
} else {
@@ -101,4 +123,125 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
confStore.confirmMutation(id, true);
}
+
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..1f94c1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A configuration mutation ACL policy which checks that user has admin
+ * privileges on all queues they are changing.
+ */
+public class QueueAdminConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private RMContext rmContext;
+
+ @Override
+ public void init(Configuration conf, RMContext context) {
+ this.rmContext = context;
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ Set<String> queues = new HashSet<>();
+ for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
+ queues.add(addQueueInfo.getQueue());
+ }
+ for (String removeQueue : confUpdate.getRemoveQueueInfo()) {
+ queues.add(removeQueue);
+ }
+ for (QueueConfigInfo updateQueueInfo : confUpdate.getUpdateQueueInfo()) {
+ queues.add(updateQueueInfo.getQueue());
+ }
+ for (String queuePath : queues) {
+ String queueName = queuePath.lastIndexOf('.') != -1 ?
+ queuePath.substring(queuePath.lastIndexOf('.') + 1) : queuePath;
+ QueueInfo queueInfo = null;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ String parentPath = queuePath;
+ // TODO: handle global config change.
+ while (queueInfo == null) {
+ // We are adding a queue (whose parent we are possibly also adding).
+ // Check ACL of lowest parent queue which already exists.
+ parentPath = parentPath.substring(0, parentPath.lastIndexOf('.'));
+ String parentName = parentPath.lastIndexOf('.') != -1 ?
+ parentPath.substring(parentPath.lastIndexOf('.') + 1) : parentPath;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(parentName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ }
+ Queue queue = ((MutableConfScheduler) rmContext.getScheduler())
+ .getQueue(queueInfo.getQueueName());
+ if (queue != null && !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, user)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 215e511..d95465b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -142,7 +142,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2484,10 +2483,8 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException, YarnException {
- Map<String, String> confUpdate =
- constructKeyValueConfUpdate(mutationInfo);
- ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
- confUpdate);
+ ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI,
+ mutationInfo);
return null;
}
});
@@ -2499,129 +2496,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
"successfully applied.").build();
} else {
return Response.status(Status.BAD_REQUEST)
- .entity("Configuration change only supported by CapacityScheduler.")
+ .entity("Configuration change only supported by " +
+ "MutableConfScheduler.")
.build();
}
}
-
- private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
- CapacitySchedulerConfiguration currentConf =
- ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
- CapacitySchedulerConfiguration proposedConf =
- new CapacitySchedulerConfiguration(currentConf, false);
- Map<String, String> confUpdate = new HashMap<>();
- for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
- removeQueue(queueToRemove, proposedConf, confUpdate);
- }
- for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
- addQueue(addQueueInfo, proposedConf, confUpdate);
- }
- for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
- updateQueue(updateQueueInfo, proposedConf, confUpdate);
- }
- return confUpdate;
- }
-
- private void removeQueue(
- String queueToRemove, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (queueToRemove == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queueName = queueToRemove.substring(
- queueToRemove.lastIndexOf('.') + 1);
- CSQueue queue = cs.getQueue(queueName);
- if (queue == null ||
- !queue.getQueuePath().equals(queueToRemove)) {
- throw new IOException("Queue " + queueToRemove + " not found");
- } else if (queueToRemove.lastIndexOf('.') == -1) {
- throw new IOException("Can't remove queue " + queueToRemove);
- }
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
- List<String> newSiblingQueues = new ArrayList<>();
- for (String siblingQueue : siblingQueues) {
- if (!siblingQueue.equals(queueName)) {
- newSiblingQueues.add(siblingQueue);
- }
- }
- proposedConf.setQueues(parentQueuePath, newSiblingQueues
- .toArray(new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
- parentQueuePath + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES;
- if (newSiblingQueues.size() == 0) {
- confUpdate.put(queuesConfig, null);
- } else {
- confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
- }
- for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
- .entrySet()) {
- proposedConf.unset(confRemove.getKey());
- confUpdate.put(confRemove.getKey(), null);
- }
- }
- }
-
- private void addQueue(
- QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (addInfo == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queuePath = addInfo.getQueue();
- String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (cs.getQueue(queueName) != null) {
- throw new IOException("Can't add existing queue " + queuePath);
- } else if (queuePath.lastIndexOf('.') == -1) {
- throw new IOException("Can't add invalid queue " + queuePath);
- }
- String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
- String[] siblings = proposedConf.getQueues(parentQueue);
- List<String> siblingQueues = siblings == null ? new ArrayList<>() :
- new ArrayList<>(Arrays.<String>asList(siblings));
- siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
- proposedConf.setQueues(parentQueue,
- siblingQueues.toArray(new String[0]));
- confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
- parentQueue + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES,
- Joiner.on(',').join(siblingQueues));
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
-
- private void updateQueue(QueueConfigInfo updateInfo,
- CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) {
- if (updateInfo == null) {
- return;
- } else {
- String queuePath = updateInfo.getQueue();
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
new file mode 100644
index 0000000..4016dcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestConfigurationMutationACLPolicies {
+
+ private ConfigurationMutationACLPolicy policy;
+ private RMContext rmContext;
+ private MutableConfScheduler scheduler;
+
+ private static final UserGroupInformation GOOD_USER = UserGroupInformation
+ .createUserForTesting("goodUser", new String[] {});
+ private static final UserGroupInformation BAD_USER = UserGroupInformation
+ .createUserForTesting("badUser", new String[] {});
+ private static final Map<String, String> EMPTY_MAP =
+ Collections.<String, String>emptyMap();
+
+ @Before
+ public void setUp() throws IOException {
+ rmContext = mock(RMContext.class);
+ scheduler = mock(MutableConfScheduler.class);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ mockQueue("a", scheduler);
+ mockQueue("b", scheduler);
+ mockQueue("b1", scheduler);
+ }
+
+ private void mockQueue(String queueName, MutableConfScheduler scheduler)
+ throws IOException {
+ QueueInfo queueInfo = QueueInfo.newInstance(queueName, 0, 0, 0, null, null,
+ null, null, null, null, false);
+ when(scheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean()))
+ .thenReturn(queueInfo);
+ Queue queue = mock(Queue.class);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(GOOD_USER)))
+ .thenReturn(true);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(BAD_USER)))
+ .thenReturn(false);
+ when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
+ }
+ @Test
+ public void testDefaultPolicy() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, null));
+ assertFalse(policy.isMutationAllowed(BAD_USER, null));
+ }
+
+ @Test
+ public void testQueueAdminBasedPolicy() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddNestedQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1.b11. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyRemoveQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Remove root.b.b1.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b.b1");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4e62530/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 254da31..13229b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -19,8 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -43,22 +47,34 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private Map<String, String> goodUpdate;
- private Map<String, String> badUpdate;
+ private QueueConfigsUpdateInfo goodUpdate;
+ private QueueConfigsUpdateInfo badUpdate;
private CapacityScheduler cs;
- private static final String TEST_USER = "testUser";
+ private static final UserGroupInformation TEST_USER = UserGroupInformation
+ .createUserForTesting("testUser", new String[] {});
@Before
public void setUp() {
cs = mock(CapacityScheduler.class);
rmContext = mock(RMContext.class);
when(rmContext.getScheduler()).thenReturn(cs);
+ when(cs.getConfiguration()).thenReturn(
+ new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new HashMap<>();
- goodUpdate.put("goodKey", "goodVal");
- badUpdate = new HashMap<>();
- badUpdate.put("badKey", "badVal");
+ goodUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> goodUpdateMap = new HashMap<>();
+ goodUpdateMap.put("goodKey", "goodVal");
+ QueueConfigInfo goodUpdateInfo = new
+ QueueConfigInfo("root.a", goodUpdateMap);
+ goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
+
+ badUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> badUpdateMap = new HashMap<>();
+ badUpdateMap.put("badKey", "badVal");
+ QueueConfigInfo badUpdateInfo = new
+ QueueConfigInfo("root.a", badUpdateMap);
+ badUpdate.getUpdateQueueInfo().add(badUpdateInfo);
}
@Test
@@ -66,15 +82,16 @@ public class TestMutableCSConfigurationProvider {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
doNothing().when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
confProvider.mutateConfiguration(TEST_USER, goodUpdate);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
try {
@@ -82,6 +99,7 @@ public class TestMutableCSConfigurationProvider {
} catch (IOException e) {
// Expected exception.
}
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-7251. Misc changes to
YARN-5734
Posted by su...@apache.org.
YARN-7251. Misc changes to YARN-5734
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09c5dfe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09c5dfe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09c5dfe9
Branch: refs/heads/YARN-5881
Commit: 09c5dfe937f0570cd9494b34d210df2d5f0737a7
Parents: a3075de
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Sep 25 17:52:40 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 3 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 33 ++++++++++++++-
.../hadoop/yarn/client/cli/SchedConfCLI.java | 42 ++++++++++++++------
.../yarn/client/cli/TestSchedConfCLI.java | 20 +++++-----
.../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 2 +-
.../yarn/webapp/dao/SchedConfUpdateInfo.java | 2 +-
.../src/main/resources/yarn-default.xml | 15 ++++---
.../scheduler/capacity/CapacityScheduler.java | 12 +++---
.../CapacitySchedulerConfiguration.java | 12 ------
.../conf/MutableCSConfigurationProvider.java | 2 +-
.../resourcemanager/webapp/RMWebServices.java | 2 +-
.../resourcemanager/TestRMAdminService.java | 4 +-
.../conf/TestLeveldbConfigurationStore.java | 2 -
.../TestMutableCSConfigurationProvider.java | 3 ++
.../capacity/conf/TestZKConfigurationStore.java | 2 -
.../TestRMWebServicesConfigurationMutation.java | 28 ++++++-------
.../src/site/markdown/CapacityScheduler.md | 11 ++---
.../src/site/markdown/ResourceManagerRest.md | 40 ++++++++++---------
.../src/site/markdown/YarnCommands.md | 12 +++---
20 files changed, 146 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 0063596..0f19989 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,6 +47,7 @@ function hadoop_usage
hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
+ hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
@@ -142,7 +143,7 @@ function yarncmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
;;
- schedconf)
+ schedulerconf)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
;;
scmadmin)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 7ec9848..fed3d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -295,7 +295,7 @@ goto :eof
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
-:schedconf
+:schedulerconf
set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
@@ -345,6 +345,7 @@ goto :eof
@echo node prints node report(s)
@echo queue prints queue information
@echo logs dump container logs
+ @echo schedulerconf updates scheduler configuration
@echo classpath prints the class path needed to get the
@echo Hadoop jar and the required libraries
@echo daemonlog get/set the log level for each daemon
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8809a7c..ac8ff12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -674,33 +674,64 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ @Private
+ @Unstable
public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
YARN_PREFIX + "scheduler.configuration.store.class";
+ @Private
+ @Unstable
+ public static final String FILE_CONFIGURATION_STORE = "file";
+ @Private
+ @Unstable
public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ @Private
+ @Unstable
public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+ @Private
+ @Unstable
public static final String ZK_CONFIGURATION_STORE = "zk";
+ @Private
+ @Unstable
public static final String DEFAULT_CONFIGURATION_STORE =
- MEMORY_CONFIGURATION_STORE;
+ FILE_CONFIGURATION_STORE;
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+ "scheduler.configuration.leveldb-store.path";
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
YARN_PREFIX
+ "scheduler.configuration.leveldb-store.compaction-interval-secs";
+ @Private
+ @Unstable
public static final long
DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_MAX_LOGS =
YARN_PREFIX + "scheduler.configuration.store.max-logs";
+ @Private
+ @Unstable
public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
+ @Private
+ @Unstable
public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
/** Parent znode path under which ZKConfigurationStore will create znodes. */
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX
+ "scheduler.configuration.zk-store.parent-path";
+ @Private
+ @Unstable
public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH =
"/confstore";
+ @Private
+ @Unstable
public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index e17062e..11bfdd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -27,7 +27,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -48,7 +48,7 @@ import java.util.Map;
* CLI for modifying scheduler configuration.
*/
@Public
-@Evolving
+@Unstable
public class SchedConfCLI extends Configured implements Tool {
private static final String ADD_QUEUES_OPTION = "addQueues";
@@ -135,7 +135,7 @@ public class SchedConfCLI extends Configured implements Tool {
WebResource webResource = webServiceClient.resource(WebAppUtils.
getRMWebAppURLWithScheme(getConf()));
ClientResponse response = webResource.path("ws").path("v1").path("cluster")
- .path("sched-conf").accept(MediaType.APPLICATION_JSON)
+ .path("scheduler-conf").accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -170,7 +170,7 @@ public class SchedConfCLI extends Configured implements Tool {
if (args == null) {
return;
}
- List<String> queuesToRemove = Arrays.asList(args.split(","));
+ List<String> queuesToRemove = Arrays.asList(args.split(";"));
updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove));
}
@@ -199,11 +199,14 @@ public class SchedConfCLI extends Configured implements Tool {
}
private QueueConfigInfo getQueueConfigInfo(String arg) {
- String[] queueArgs = arg.split(",");
- String queuePath = queueArgs[0];
+ String[] args = arg.split(":");
+ String queuePath = args[0];
Map<String, String> queueConfigs = new HashMap<>();
- for (int i = 1; i < queueArgs.length; ++i) {
- putKeyValuePair(queueConfigs, queueArgs[i]);
+ if (args.length > 1) {
+ String[] queueArgs = args[1].split(",");
+ for (int i = 0; i < queueArgs.length; ++i) {
+ putKeyValuePair(queueConfigs, queueArgs[i]);
+ }
}
return new QueueConfigInfo(queuePath, queueConfigs);
}
@@ -228,11 +231,24 @@ public class SchedConfCLI extends Configured implements Tool {
}
private void printUsage() {
- System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1,"
- + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] "
- + "[-remove queueRemovePath1,queueRemovePath2] "
- + "[-update queueUpdatePath1,confKey1=confVal1] "
+ System.out.println("yarn schedulerconf [-add "
+ + "\"queueAddPath1:confKey1=confVal1,confKey2=confVal2;"
+ + "queueAddPath2:confKey3=confVal3\"] "
+ + "[-remove \"queueRemovePath1;queueRemovePath2\"] "
+ + "[-update \"queueUpdatePath1:confKey1=confVal1\"] "
+ "[-global globalConfKey1=globalConfVal1,"
- + "globalConfKey2=globalConfVal2]");
+ + "globalConfKey2=globalConfVal2]\n"
+ + "Example (adding queues): yarn schedulerconf -add "
+ + "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ + "maximum-capacity=0\"\n"
+ + "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;"
+ + "root.a.a2\"\n"
+ + "Example (updating queues): yarn schedulerconf -update \"root.a.a1"
+ + ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75,"
+ + "maximum-capacity=75\"\n"
+ + "Example (global scheduler update): yarn schedulerconf "
+ + "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ + "Note: This is an alpha feature, the syntax/options are subject to "
+ + "change, please run at your own risk.");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index d2f0639..5364e83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -61,20 +61,20 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testInvalidConf() throws Exception {
// conf pair with no key should be invalid
- int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"});
+ int exitCode = cli.run(new String[] {"-add", "root.a:=confVal"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
- exitCode = cli.run(new String[] {"-update", "root.a,=confVal"});
+ exitCode = cli.run(new String[] {"-update", "root.a:=confVal"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
- exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"});
+ exitCode = cli.run(new String[] {"-add", "root.a:confKey=confVal=conf"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
- exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"});
+ exitCode = cli.run(new String[] {"-update", "root.a:confKey=confVal=c"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
@@ -83,8 +83,7 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testAddQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
- cli.addQueues("root.a,a1=aVal1,a2=aVal2," +
- "a3=", schedUpdateInfo);
+ cli.addQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo);
QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0);
assertEquals("root.a", addInfo.getQueue());
Map<String, String> params = addInfo.getParams();
@@ -94,7 +93,7 @@ public class TestSchedConfCLI {
assertNull(params.get("a3"));
schedUpdateInfo = new SchedConfUpdateInfo();
- cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ cli.addQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo);
assertEquals(2, schedUpdateInfo.getAddQueueInfo().size());
QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0);
assertEquals("root.b", bAddInfo.getQueue());
@@ -111,7 +110,7 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testRemoveQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
- cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo);
+ cli.removeQueues("root.a;root.b;root.c.c1", schedUpdateInfo);
List<String> removeInfo = schedUpdateInfo.getRemoveQueueInfo();
assertEquals(3, removeInfo.size());
assertEquals("root.a", removeInfo.get(0));
@@ -122,8 +121,7 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testUpdateQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
- cli.updateQueues("root.a,a1=aVal1,a2=aVal2," +
- "a3=", schedUpdateInfo);
+ cli.updateQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo);
QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
assertEquals("root.a", updateInfo.getQueue());
Map<String, String> params = updateInfo.getParams();
@@ -133,7 +131,7 @@ public class TestSchedConfCLI {
assertNull(params.get("a3"));
schedUpdateInfo = new SchedConfUpdateInfo();
- cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ cli.updateQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo);
assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size());
QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
assertEquals("root.b", bUpdateInfo.getQueue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
index d1d91c2..6d4e0cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
@@ -34,7 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement;
@XmlAccessorType(XmlAccessType.FIELD)
public class QueueConfigInfo {
- @XmlElement(name = "queueName")
+ @XmlElement(name = "queue-name")
private String queue;
private HashMap<String, String> params = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
index bb84096..4546291 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
@@ -31,7 +31,7 @@ import javax.xml.bind.annotation.XmlRootElement;
* Information for making scheduler configuration changes (supports adding,
* removing, or updating a queue, as well as global scheduler conf changes).
*/
-@XmlRootElement(name = "schedConf")
+@XmlRootElement(name = "sched-conf")
@XmlAccessorType(XmlAccessType.FIELD)
public class SchedConfUpdateInfo {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4516edf..421bccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3369,14 +3369,17 @@
<property>
<description>
- The type of configuration store to use for storing scheduler
- configurations, if using a mutable configuration provider.
- Keywords such as "memory" map to certain configuration store
- implementations. If keyword is not found, try to load this
- value as a class.
+ The type of configuration store to use for scheduler configurations.
+ Default is "file", which uses file based capacity-scheduler.xml to
+ retrieve and change scheduler configuration. To enable API based
+ scheduler configuration, use either "memory" (in memory storage, no
+ persistence across restarts), "leveldb" (leveldb based storage), or
+ "zk" (zookeeper based storage). API based configuration is only useful
+ when using a scheduler which supports mutable configuration. Currently
+ only capacity scheduler supports this.
</description>
<name>yarn.scheduler.configuration.store.class</name>
- <value>memory</value>
+ <value>file</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index de95179..d91aa55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -296,18 +296,20 @@ public class CapacityScheduler extends
try {
writeLock.lock();
String confProviderStr = configuration.get(
- CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
switch (confProviderStr) {
- case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER:
+ case YarnConfiguration.FILE_CONFIGURATION_STORE:
this.csConfProvider =
new FileBasedCSConfigurationProvider(rmContext);
break;
- case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER:
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
break;
default:
- throw new IOException("Invalid CS configuration provider: " +
+ throw new IOException("Invalid configuration store class: " +
confProviderStr);
}
this.csConfProvider.init(configuration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 40cb893..3a519ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -315,18 +315,6 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
-
- public static final String CS_CONF_PROVIDER = PREFIX
- + "configuration.provider";
-
- @Private
- public static final String FILE_CS_CONF_PROVIDER = "file";
-
- @Private
- public static final String STORE_CS_CONF_PROVIDER = "store";
-
- @Private
- public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ccadf76..40a19a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -66,7 +66,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
public void init(Configuration config) throws IOException {
String store = config.get(
YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
- YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
switch (store) {
case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
this.confStore = new InMemoryConfigurationStore();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index d264c10..e6a0cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2461,7 +2461,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
@PUT
- @Path("/sched-conf")
+ @Path("/scheduler-conf")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 620ca58..03fc081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -199,8 +199,8 @@ public class TestRMAdminService {
@Test
public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() {
- configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ configuration.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
try {
rm = new MockRM(configuration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
index 779208a..324cbee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -59,8 +59,6 @@ public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
public void setUp() throws Exception {
super.setUp();
FileUtil.fullyDelete(TEST_DIR);
- conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.LEVELDB_CONFIGURATION_STORE);
conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 9b080cd..5d43ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -80,6 +81,8 @@ public class TestMutableCSConfigurationProvider {
@Test
public void testInMemoryBackedProvider() throws Exception {
Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 3cfa8da..355f741 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -199,8 +199,6 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
- conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.ZK_CONFIGURATION_STORE);
conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 26ef1b7..3d28f12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -88,8 +88,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
- conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
try {
if (CONF_FILE.exists()) {
@@ -179,7 +179,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getAddQueueInfo().add(d);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -214,7 +214,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -240,7 +240,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getRemoveQueueInfo().add("root.a.a2");
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -265,7 +265,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getRemoveQueueInfo().add("root.c");
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -296,7 +296,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -328,7 +328,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(configInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -356,7 +356,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -391,7 +391,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
0.001f);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -409,7 +409,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -439,7 +439,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -464,7 +464,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -479,7 +479,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
// Unset maximum-applications. Should be set to default.
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index d70f891..17a6ab9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -295,7 +295,7 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach
Changing Queue Configuration
----------------------------
-Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API.
+Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. This behavior can be changed via `yarn.scheduler.configuration.store.class` in yarn-site.xml. Possible values are *file*, which allows modifying properties via file; *memory*, which allows modifying properties via API, but does not persist changes across restart; *leveldb*, which allows modifying properties via API and stores changes in leveldb backing store; and *zk*, which allows modifying properties via API and stores changes in zookeeper backing store. The default value is *file*.
### Changing queue configuration via file
@@ -308,16 +308,17 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw
Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml.
+ **Note:** This feature is in alpha phase and is subject to change.
+
| Property | Description |
|:---- |:---- |
- | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. |
- | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). |
+ | `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). |
| `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. |
| `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. |
| `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. |
| `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. |
| `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. |
- **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
+ **Note:** When enabling scheduler configuration mutations via `yarn.scheduler.configuration.store.class`, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
- See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line.
+ See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedulerconf) for examples on how to change scheduler configuration via command line.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index 025e9a7..f3a1907 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -4436,16 +4436,18 @@ Scheduler Configuration Mutation API
The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy.
+Please note that this feature is currently in the alpha stage and is subject to change.
+
### URI
- * http://rm-http-address:port/ws/v1/cluster/sched-conf
+ * http://rm-http-address:port/ws/v1/cluster/scheduler-conf
### HTTP Operations Supported
* PUT
-### Elements of the *schedConf* object
+### Elements of the *sched-conf* object
| Item | Data Type | Description |
|:---- |:---- |:---- |
@@ -4466,7 +4468,7 @@ Request for updating queue configurations.
| Item | Data Type | Description |
|:---- |:---- |:---- |
-| queueName | string | Full path name of the queue to update |
+| queue-name | string | Full path name of the queue to update |
| params | map | A map of key value configuration pairs to update for this queue |
Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10.
@@ -4475,12 +4477,12 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<update-queue>
- <queueName>root.default</queueName>
+ <queue-name>root.default</queue-name>
<params>
<entry>
<key>maximum-applications</key>
@@ -4492,7 +4494,7 @@ HTTP Request:
</entry>
</params>
</update-queue>
- </schedConf>
+ </sched-conf>
```
@@ -4511,7 +4513,7 @@ Request for adding queues/updating queue configurations.
| Item | Data Type | Description |
|:---- |:---- |:---- |
-| queueName | string | Full path name of the queue to add |
+| queue-name | string | Full path name of the queue to add |
| params | map | A map of key value configuration pairs to set for this queue |
Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.)
@@ -4520,12 +4522,12 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<add-queue>
- <queueName>root.a</queueName>
+ <queue-name>root.a</queue-name>
<params>
<entry>
<key>capacity</key>
@@ -4538,7 +4540,7 @@ HTTP Request:
</params>
</add-queue>
<update-queue>
- <queueName>root.default</queueName>
+ <queue-name>root.default</queue-name>
<params>
<entry>
<key>capacity</key>
@@ -4550,7 +4552,7 @@ HTTP Request:
</entry>
</params>
</update-queue>
- </schedConf>
+ </sched-conf>
```
@@ -4573,13 +4575,13 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<remove-queue>root.a</remove-queue>
<remove-queue>root.b</remove-queue>
- </schedConf>
+ </sched-conf>
```
@@ -4597,17 +4599,17 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<global-updates>
<entry>
<key>yarn.scheduler.capacity.queue-mappings-override.enable</key>
<value>true</value>
</entry>
</global-updates>
- </schedConf>
+ </sched-conf>
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09c5dfe9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 2fc65bc..fa33b8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -237,18 +237,18 @@ Usage:
Runs ResourceManager admin client
-### schedconf
+### schedulerconf
-Usage: `yarn schedconf [options]`
+Usage: `yarn schedulerconf [options]`
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
-| -add <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. |
-| -remove <queuePath1,queuePath2> | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. |
-| -update <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. |
+| -add <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. |
+| -remove <"queuePath1;queuePath2"> | Semicolon separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. |
+| -update <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. |
| -global <key1=val1,key2=val2> | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. |
-Updates scheduler configuration
+Updates scheduler configuration. Note, this feature is in alpha phase and is subject to change.
### scmadmin
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HADOOP-13514. Upgrade maven
surefire plugin to 2.19.1.
Posted by su...@apache.org.
HADOOP-13514. Upgrade maven surefire plugin to 2.19.1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d04c00a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d04c00a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d04c00a
Branch: refs/heads/YARN-5881
Commit: 3d04c00aed63ca49420210f5f34efe39897aae63
Parents: a297fb0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 11 10:42:32 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 11 10:42:32 2017 +0900
----------------------------------------------------------------------
BUILDING.txt | 4 ++--
dev-support/docker/Dockerfile | 2 +-
hadoop-project/pom.xml | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d04c00a/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 47aaab4..9955563 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -331,10 +331,10 @@ If the build process fails with an out of memory error, you should be able to fi
it by increasing the memory used by maven which can be done via the environment
variable MAVEN_OPTS.
-Here is an example setting to allocate between 256 and 512 MB of heap space to
+Here is an example setting to allocate between 256 MB and 1 GB of heap space to
Maven
-export MAVEN_OPTS="-Xms256m -Xmx512m"
+export MAVEN_OPTS="-Xms256m -Xmx1g"
----------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d04c00a/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 31ac611..1ced9ef 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -147,7 +147,7 @@ RUN pip install python-dateutil
###
# Avoid out of memory errors in builds
###
-ENV MAVEN_OPTS -Xms256m -Xmx512m
+ENV MAVEN_OPTS -Xms256m -Xmx1g
###
# Install node js tools for web UI frameowkr
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d04c00a/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8980f0e..66e3dfc 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -117,7 +117,7 @@
<!-- Plugin versions and config -->
<maven-surefire-plugin.argLine>-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
- <maven-surefire-plugin.version>2.17</maven-surefire-plugin.version>
+ <maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version>
<maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
<maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
@@ -1546,7 +1546,7 @@
<test.build.data>${test.build.data}</test.build.data>
<test.build.webapps>${test.build.webapps}</test.build.webapps>
<test.cache.data>${test.cache.data}</test.cache.data>
- <test.build.classes>${test.build.classes}</test.build.classes>
+ <test.build.classes>${project.build.directory}/test-classes</test.build.classes>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
<java.security.krb5.conf>${project.build.directory}/test-classes/krb5.conf</java.security.krb5.conf>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-14939. Update project
release notes with HDFS-10467 for 3.0.0. Contributed by Inigo Goiri.
Posted by su...@apache.org.
HADOOP-14939. Update project release notes with HDFS-10467 for 3.0.0. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132cdac0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132cdac0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132cdac0
Branch: refs/heads/YARN-5881
Commit: 132cdac0ddb5c38205a96579a23b55689ea5a8e3
Parents: 41351b0
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Oct 10 10:16:36 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Oct 10 10:16:36 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/markdown/index.md.vm | 15 +++++++++++++++
1 file changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132cdac0/hadoop-project/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
index d9e645b..8e1e06f 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -213,6 +213,21 @@ metadata.
See [S3Guard](./hadoop-aws/tools/hadoop-aws/s3guard.html) for more details.
+HDFS Router-Based Federation
+---------------------
+HDFS Router-Based Federation adds a RPC routing layer that provides a federated
+view of multiple HDFS namespaces. This is similar to the existing
+[ViewFs](./hadoop-project-dist/hadoop-hdfs/ViewFs.html)) and
+[HDFS Federation](./hadoop-project-dist/hadoop-hdfs/Federation.html)
+functionality, except the mount table is managed on the server-side by the
+routing layer rather than on the client. This simplifies access to a federated
+cluster for existing HDFS clients.
+
+See [HDFS-10467](https://issues.apache.org/jira/browse/HADOOP-10467) and the
+HDFS Router-based Federation
+[documentation](./hadoop-project-dist/hadoop-hdfs/HDFSRouterFederation.md) for
+more details.
+
Getting Started
===============
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-7082.
TestContainerManagerSecurity failing in trunk. Contributed by Akira Ajisaka
Posted by su...@apache.org.
YARN-7082. TestContainerManagerSecurity failing in trunk. Contributed by Akira Ajisaka
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebb34c70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebb34c70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebb34c70
Branch: refs/heads/YARN-5881
Commit: ebb34c7053ee93296412591f83a776846765fda5
Parents: dc63a6a
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Oct 11 10:31:02 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Oct 11 10:31:02 2017 -0500
----------------------------------------------------------------------
.../server/security/BaseContainerTokenSecretManager.java | 8 +++++---
.../hadoop/yarn/server/ContainerTokenIdentifierForTest.java | 7 +++++++
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb34c70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
index a5590fb..420c899 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
@@ -112,9 +112,11 @@ public class BaseContainerTokenSecretManager extends
protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
MasterKeyData masterKey)
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
- LOG.debug("Retrieving password for {} for user {} to be run on NM {}",
- identifier.getContainerID(), identifier.getUser(),
- identifier.getNmHostAddress());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrieving password for {} for user {} to be run on NM {}",
+ identifier.getContainerID(), identifier.getUser(),
+ identifier.getNmHostAddress());
+ }
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb34c70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
index 0d4d2e2..068b140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -121,6 +122,12 @@ public class ContainerTokenIdentifierForTest extends ContainerTokenIdentifier {
return new ContainerIdPBImpl(proto.getContainerId());
}
+ @Override
+ public UserGroupInformation getUser() {
+ final ContainerId containerId = getContainerID();
+ return UserGroupInformation.createRemoteUser(containerId.toString());
+ }
+
public String getApplicationSubmitter() {
return proto.getAppSubmitter();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDFS-11575. Supporting HDFS NFS
gateway with Federated HDFS. Contributed by Mukul Kumar Singh.
Posted by su...@apache.org.
HDFS-11575. Supporting HDFS NFS gateway with Federated HDFS. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6602b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6602b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6602b5f
Branch: refs/heads/YARN-5881
Commit: d6602b5f39833611b4afa4581552f6c4c37e23a8
Parents: ec8bf9e
Author: Jitendra Pandey <ji...@apache.org>
Authored: Tue Oct 10 09:49:46 2017 -0700
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Tue Oct 10 10:38:05 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/nfs/nfs3/FileHandle.java | 51 ++-
.../hadoop/nfs/nfs3/request/WRITE3Request.java | 4 +-
.../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 81 +++--
.../hadoop/hdfs/nfs/nfs3/DFSClientCache.java | 174 ++++++---
.../apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 46 +++
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 19 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 6 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 362 +++++++++++--------
.../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 8 +-
.../hadoop/hdfs/nfs/nfs3/WriteManager.java | 24 +-
.../nfs/nfs3/TestClientAccessPrivilege.java | 3 +-
.../hdfs/nfs/nfs3/TestDFSClientCache.java | 13 +-
.../hadoop/hdfs/nfs/nfs3/TestExportsTable.java | 161 ++++++++-
.../hadoop/hdfs/nfs/nfs3/TestReaddir.java | 19 +-
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 66 ++--
.../hdfs/nfs/nfs3/TestViewfsWithNfs3.java | 330 +++++++++++++++++
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 9 +-
17 files changed, 1071 insertions(+), 305 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
index 5b32798..910b8f2 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
@@ -38,17 +38,21 @@ public class FileHandle {
private static final int HANDLE_LEN = 32;
private byte[] handle; // Opaque handle
private long fileId = -1;
+ private int namenodeId = -1;
public FileHandle() {
handle = null;
}
/**
- * Handle is a 32 bytes number. For HDFS, the last 8 bytes is fileId.
+ * Handle is a 32 bytes number. For HDFS, the last 8 bytes is fileId
+ * For ViewFs, last 8 byte is fileId while 4 bytes before that is namenodeId
* @param v file id
+ * @param n namenode id
*/
- public FileHandle(long v) {
+ public FileHandle(long v, int n) {
fileId = v;
+ namenodeId = n;
handle = new byte[HANDLE_LEN];
handle[0] = (byte)(v >>> 56);
handle[1] = (byte)(v >>> 48);
@@ -58,11 +62,20 @@ public class FileHandle {
handle[5] = (byte)(v >>> 16);
handle[6] = (byte)(v >>> 8);
handle[7] = (byte)(v >>> 0);
- for (int i = 8; i < HANDLE_LEN; i++) {
+
+ handle[8] = (byte) (n >>> 24);
+ handle[9] = (byte) (n >>> 16);
+ handle[10] = (byte) (n >>> 8);
+ handle[11] = (byte) (n >>> 0);
+ for (int i = 12; i < HANDLE_LEN; i++) {
handle[i] = (byte) 0;
}
}
-
+
+ public FileHandle(long v) {
+ this(v, 0);
+ }
+
public FileHandle(String s) {
MessageDigest digest;
try {
@@ -93,22 +106,32 @@ public class FileHandle {
return true;
}
- private long bytesToLong(byte[] data) {
+ private long bytesToLong(byte[] data, int offset) {
ByteBuffer buffer = ByteBuffer.allocate(8);
for (int i = 0; i < 8; i++) {
- buffer.put(data[i]);
+ buffer.put(data[i + offset]);
}
- buffer.flip();// need flip
+ buffer.flip(); // need flip
return buffer.getLong();
}
-
+
+ private int bytesToInt(byte[] data, int offset) {
+ ByteBuffer buffer = ByteBuffer.allocate(4);
+ for (int i = 0; i < 4; i++) {
+ buffer.put(data[i + offset]);
+ }
+ buffer.flip(); // need flip
+ return buffer.getInt();
+ }
+
public boolean deserialize(XDR xdr) {
if (!XDR.verifyLength(xdr, 32)) {
return false;
}
int size = xdr.readInt();
handle = xdr.readFixedOpaque(size);
- fileId = bytesToLong(handle);
+ fileId = bytesToLong(handle, 0);
+ namenodeId = bytesToInt(handle, 8);
return true;
}
@@ -122,11 +145,15 @@ public class FileHandle {
public long getFileId() {
return fileId;
}
+
+ public int getNamenodeId() {
+ return namenodeId;
+ }
public byte[] getContent() {
return handle.clone();
}
-
+
@Override
public String toString() {
StringBuilder s = new StringBuilder();
@@ -154,4 +181,8 @@ public class FileHandle {
public int hashCode() {
return Arrays.hashCode(handle);
}
+
+ public String dumpFileHandle() {
+ return "fileId: " + fileId + " namenodeId: " + namenodeId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java
index d85dcbb..86a40c7 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/WRITE3Request.java
@@ -87,7 +87,7 @@ public class WRITE3Request extends RequestWithHandle {
@Override
public String toString() {
- return String.format("fileId: %d offset: %d count: %d stableHow: %s",
- handle.getFileId(), offset, count, stableHow.name());
+ return String.format("fileHandle: %s offset: %d count: %d stableHow: %s",
+ handle.dumpFileHandle(), offset, count, stableHow.name());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
index e31bc71..4ae51c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
@@ -19,17 +19,20 @@ package org.apache.hadoop.hdfs.nfs.mount;
import java.io.IOException;
import java.net.DatagramSocket;
import java.net.InetAddress;
+import java.net.URI;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
+import org.apache.hadoop.hdfs.nfs.nfs3.Nfs3Utils;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.mount.MountEntry;
import org.apache.hadoop.mount.MountInterface;
@@ -64,14 +67,12 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
public static final int VERSION_2 = 2;
public static final int VERSION_3 = 3;
- private final DFSClient dfsClient;
-
- /** Synchronized list */
+ /** Synchronized list. */
private final List<MountEntry> mounts;
- /** List that is unmodifiable */
- private final List<String> exports;
-
+ /** List that is unmodifiable. */
+ private final HashMap<String, URI> exports;
+ private final NfsConfiguration config;
private final NfsExports hostsMatcher;
public RpcProgramMountd(NfsConfiguration config,
@@ -84,17 +85,29 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
VERSION_3, registrationSocket, allowInsecurePorts, config.getInt(
NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_DEFAULT));
- exports = new ArrayList<String>();
- exports.add(config.get(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY,
- NfsConfigKeys.DFS_NFS_EXPORT_POINT_DEFAULT));
+ this.config = config;
+ exports = new HashMap<>();
+ addExports();
this.hostsMatcher = NfsExports.getInstance(config);
this.mounts = Collections.synchronizedList(new ArrayList<MountEntry>());
UserGroupInformation.setConfiguration(config);
SecurityUtil.login(config, NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY,
NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY);
- this.dfsClient = new DFSClient(DFSUtilClient.getNNAddress(config), config);
}
-
+
+ private void addExports() throws IOException {
+ FileSystem fs = FileSystem.get(config);
+ String[] exportsPath =
+ config.getStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY,
+ NfsConfigKeys.DFS_NFS_EXPORT_POINT_DEFAULT);
+ for (String exportPath : exportsPath) {
+ URI exportURI = Nfs3Utils.getResolvedURI(fs, exportPath);
+ LOG.info("FS:" + fs.getScheme() + " adding export Path:" + exportPath +
+ " with URI: " + exportURI.toString());
+ exports.put(exportPath, exportURI);
+ }
+ }
+
@Override
public XDR nullOp(XDR out, int xid, InetAddress client) {
if (LOG.isDebugEnabled()) {
@@ -125,17 +138,28 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
if (LOG.isDebugEnabled()) {
LOG.debug("Got host: " + host + " path: " + path);
}
- if (!exports.contains(path)) {
+ URI exportURI = exports.get(path);
+ if (exportURI == null) {
LOG.info("Path " + path + " is not shared.");
MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_NOENT, out, xid, null);
return out;
}
+ DFSClient dfsClient = null;
+ try {
+ dfsClient = new DFSClient(exportURI, config);
+ } catch (Exception e) {
+ LOG.error("Can't get handle for export:" + path, e);
+ MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_NOENT, out, xid, null);
+ return out;
+ }
+
FileHandle handle = null;
try {
- HdfsFileStatus exFileStatus = dfsClient.getFileInfo(path);
-
- handle = new FileHandle(exFileStatus.getFileId());
+ HdfsFileStatus exFileStatus = dfsClient.getFileInfo(exportURI.getPath());
+
+ handle = new FileHandle(exFileStatus.getFileId(),
+ Nfs3Utils.getNamenodeId(config, exportURI));
} catch (IOException e) {
LOG.error("Can't get handle for export:" + path, e);
MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_NOENT, out, xid, null);
@@ -143,7 +167,8 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
}
assert (handle != null);
- LOG.info("Giving handle (fileId:" + handle.getFileId()
+ LOG.info("Giving handle (fileHandle:" + handle.dumpFileHandle()
+ + " file URI: " + exportURI
+ ") to client for export " + path);
mounts.add(new MountEntry(host, path));
@@ -195,7 +220,8 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
info.data().readBytes(data);
XDR xdr = new XDR(data);
XDR out = new XDR();
- InetAddress client = ((InetSocketAddress) info.remoteAddress()).getAddress();
+ InetAddress client =
+ ((InetSocketAddress) info.remoteAddress()).getAddress();
if (mntproc == MNTPROC.NULL) {
out = nullOp(out, xid, client);
@@ -214,16 +240,20 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
} else if (mntproc == MNTPROC.UMNTALL) {
umntall(out, xid, client);
} else if (mntproc == MNTPROC.EXPORT) {
- // Currently only support one NFS export
+ // Currently only support one NFS export per namenode
List<NfsExports> hostsMatchers = new ArrayList<NfsExports>();
if (hostsMatcher != null) {
- hostsMatchers.add(hostsMatcher);
- out = MountResponse.writeExportList(out, xid, exports, hostsMatchers);
+ List exportsList = getExports();
+ for (int i = 0; i < exportsList.size(); i++) {
+ hostsMatchers.add(hostsMatcher);
+ }
+ out = MountResponse.writeExportList(out, xid,
+ exportsList, hostsMatchers);
} else {
// This means there are no valid exports provided.
RpcAcceptedReply.getInstance(xid,
- RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()).write(
- out);
+ RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone())
+ .write(out);
}
} else {
// Invalid procedure
@@ -231,7 +261,8 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()).write(
out);
}
- ChannelBuffer buf = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap().buffer());
+ ChannelBuffer buf =
+ ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap().buffer());
RpcResponse rsp = new RpcResponse(buf, info.remoteAddress());
RpcUtil.sendRpcResponse(ctx, rsp);
}
@@ -244,6 +275,6 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface {
@VisibleForTesting
public List<String> getExports() {
- return this.exports;
+ return new ArrayList<>(this.exports.keySet());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
index b946bce..9a9366f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.hdfs.nfs.nfs3;
import org.apache.commons.logging.LogFactory;
import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystemException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
@@ -31,9 +34,10 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSInputStream;
-import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -48,63 +52,97 @@ import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
/**
- * A cache saves DFSClient objects for different users
+ * A cache saves DFSClient objects for different users.
*/
class DFSClientCache {
private static final Log LOG = LogFactory.getLog(DFSClientCache.class);
/**
* Cache that maps User id to the corresponding DFSClient.
*/
- @VisibleForTesting
- final LoadingCache<String, DFSClient> clientCache;
+ private final LoadingCache<DfsClientKey, DFSClient> clientCache;
final static int DEFAULT_DFS_CLIENT_CACHE_SIZE = 256;
/**
- * Cache that maps <DFSClient, inode path> to the corresponding
+ * Cache that maps <DFSClient, inode path, nnid> to the corresponding
* FSDataInputStream.
*/
- final LoadingCache<DFSInputStreamCaheKey, FSDataInputStream> inputstreamCache;
+ private final LoadingCache<DFSInputStreamCacheKey,
+ FSDataInputStream> inputstreamCache;
/**
- * Time to live for a DFSClient (in seconds)
+ * Time to live for a DFSClient (in seconds).
*/
final static int DEFAULT_DFS_INPUTSTREAM_CACHE_SIZE = 1024;
final static int DEFAULT_DFS_INPUTSTREAM_CACHE_TTL = 10 * 60;
private final NfsConfiguration config;
+ private final HashMap<Integer, URI> namenodeUriMap;
- private static class DFSInputStreamCaheKey {
- final String userId;
- final String inodePath;
+ private static final class DFSInputStreamCacheKey {
+ private final String userId;
+ private final String inodePath;
+ private final int namenodeId;
- private DFSInputStreamCaheKey(String userId, String inodePath) {
+ private DFSInputStreamCacheKey(String userId, String inodePath,
+ int namenodeId) {
super();
this.userId = userId;
this.inodePath = inodePath;
+ this.namenodeId = namenodeId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof DFSInputStreamCacheKey) {
+ DFSInputStreamCacheKey k = (DFSInputStreamCacheKey) obj;
+ return userId.equals(k.userId) &&
+ inodePath.equals(k.inodePath) &&
+ (namenodeId == k.namenodeId);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(userId, inodePath, namenodeId);
+ }
+ }
+
+ private static final class DfsClientKey {
+ private final String userName;
+ private final int namenodeId;
+
+ private DfsClientKey(String userName, int namenodeId) {
+ this.userName = userName;
+ this.namenodeId = namenodeId;
}
@Override
public boolean equals(Object obj) {
- if (obj instanceof DFSInputStreamCaheKey) {
- DFSInputStreamCaheKey k = (DFSInputStreamCaheKey) obj;
- return userId.equals(k.userId) && inodePath.equals(k.inodePath);
+ if (obj instanceof DfsClientKey) {
+ DfsClientKey k = (DfsClientKey) obj;
+ return userName.equals(k.userName) &&
+ (namenodeId == k.namenodeId);
}
return false;
}
@Override
public int hashCode() {
- return Objects.hashCode(userId, inodePath);
+ return Objects.hashCode(userName, namenodeId);
}
}
- DFSClientCache(NfsConfiguration config) {
+ DFSClientCache(NfsConfiguration config) throws IOException {
this(config, DEFAULT_DFS_CLIENT_CACHE_SIZE);
}
-
- DFSClientCache(NfsConfiguration config, int clientCache) {
+
+ DFSClientCache(NfsConfiguration config, int clientCache) throws IOException {
this.config = config;
+ namenodeUriMap = new HashMap<>();
+ prepareAddressMap();
+
this.clientCache = CacheBuilder.newBuilder()
.maximumSize(clientCache)
.removalListener(clientRemovalListener())
@@ -115,11 +153,36 @@ class DFSClientCache {
.expireAfterAccess(DEFAULT_DFS_INPUTSTREAM_CACHE_TTL, TimeUnit.SECONDS)
.removalListener(inputStreamRemovalListener())
.build(inputStreamLoader());
-
+
ShutdownHookManager.get().addShutdownHook(new CacheFinalizer(),
SHUTDOWN_HOOK_PRIORITY);
}
+ private void prepareAddressMap() throws IOException {
+ FileSystem fs = FileSystem.get(config);
+ String[] exportsPath =
+ config.getStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY,
+ NfsConfigKeys.DFS_NFS_EXPORT_POINT_DEFAULT);
+ for (String exportPath : exportsPath) {
+ URI exportURI = Nfs3Utils.getResolvedURI(fs, exportPath);
+ int namenodeId = Nfs3Utils.getNamenodeId(config, exportURI);
+ URI value = namenodeUriMap.get(namenodeId);
+ // if a unique nnid, add it to the map
+ if (value == null) {
+ LOG.info("Added export:" + exportPath + " FileSystem URI:" + exportURI
+ + " with namenodeId:" + namenodeId);
+ namenodeUriMap.put(namenodeId, exportURI);
+ } else {
+ // if the nnid already exists, it better be the for the same namenode
+ String msg = String.format("FS:%s, Namenode ID collision for path:%s "
+ + "nnid:%s uri being added:%s existing uri:%s", fs.getScheme(),
+ exportPath, namenodeId, exportURI, value);
+ LOG.error(msg);
+ throw new FileSystemException(msg);
+ }
+ }
+ }
+
/**
* Priority of the FileSystem shutdown hook.
*/
@@ -135,7 +198,12 @@ class DFSClientCache {
}
}
}
-
+
+ @VisibleForTesting
+ public LoadingCache<DfsClientKey, DFSClient> getClientCache() {
+ return clientCache;
+ }
+
/**
* Close all DFSClient instances in the Cache.
* @param onlyAutomatic only close those that are marked for automatic closing
@@ -143,9 +211,9 @@ class DFSClientCache {
synchronized void closeAll(boolean onlyAutomatic) throws IOException {
List<IOException> exceptions = new ArrayList<IOException>();
- ConcurrentMap<String, DFSClient> map = clientCache.asMap();
+ ConcurrentMap<DfsClientKey, DFSClient> map = clientCache.asMap();
- for (Entry<String, DFSClient> item : map.entrySet()) {
+ for (Entry<DfsClientKey, DFSClient> item : map.entrySet()) {
final DFSClient client = item.getValue();
if (client != null) {
try {
@@ -160,20 +228,24 @@ class DFSClientCache {
throw MultipleIOException.createIOException(exceptions);
}
}
-
- private CacheLoader<String, DFSClient> clientLoader() {
- return new CacheLoader<String, DFSClient>() {
+
+ private CacheLoader<DfsClientKey, DFSClient> clientLoader() {
+ return new CacheLoader<DfsClientKey, DFSClient>() {
@Override
- public DFSClient load(String userName) throws Exception {
+ public DFSClient load(final DfsClientKey key) throws Exception {
UserGroupInformation ugi = getUserGroupInformation(
- userName,
- UserGroupInformation.getCurrentUser());
+ key.userName, UserGroupInformation.getCurrentUser());
// Guava requires CacheLoader never returns null.
return ugi.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
- return new DFSClient(DFSUtilClient.getNNAddress(config), config);
+ URI namenodeURI = namenodeUriMap.get(key.namenodeId);
+ if (namenodeURI == null) {
+ throw new IOException("No namenode URI found for user:" +
+ key.userName + " namenodeId:" + key.namenodeId);
+ }
+ return new DFSClient(namenodeURI, config);
}
});
}
@@ -181,7 +253,7 @@ class DFSClientCache {
}
/**
- * This method uses the currentUser, and real user to create a proxy
+ * This method uses the currentUser, and real user to create a proxy.
* @param effectiveUser The user who is being proxied by the real user
* @param realUser The actual user who does the command
* @return Proxy UserGroupInformation
@@ -204,10 +276,11 @@ class DFSClientCache {
return ugi;
}
- private RemovalListener<String, DFSClient> clientRemovalListener() {
- return new RemovalListener<String, DFSClient>() {
+ private RemovalListener<DfsClientKey, DFSClient> clientRemovalListener() {
+ return new RemovalListener<DfsClientKey, DFSClient>() {
@Override
- public void onRemoval(RemovalNotification<String, DFSClient> notification) {
+ public void onRemoval(
+ RemovalNotification<DfsClientKey, DFSClient> notification) {
DFSClient client = notification.getValue();
try {
client.close();
@@ -220,12 +293,15 @@ class DFSClientCache {
};
}
- private RemovalListener<DFSInputStreamCaheKey, FSDataInputStream> inputStreamRemovalListener() {
- return new RemovalListener<DFSClientCache.DFSInputStreamCaheKey, FSDataInputStream>() {
+ private RemovalListener
+ <DFSInputStreamCacheKey, FSDataInputStream> inputStreamRemovalListener() {
+ return new RemovalListener
+ <DFSClientCache.DFSInputStreamCacheKey, FSDataInputStream>() {
@Override
public void onRemoval(
- RemovalNotification<DFSInputStreamCaheKey, FSDataInputStream> notification) {
+ RemovalNotification<DFSInputStreamCacheKey, FSDataInputStream>
+ notification) {
try {
notification.getValue().close();
} catch (IOException ignored) {
@@ -234,22 +310,24 @@ class DFSClientCache {
};
}
- private CacheLoader<DFSInputStreamCaheKey, FSDataInputStream> inputStreamLoader() {
- return new CacheLoader<DFSInputStreamCaheKey, FSDataInputStream>() {
+ private CacheLoader<DFSInputStreamCacheKey, FSDataInputStream>
+ inputStreamLoader() {
+ return new CacheLoader<DFSInputStreamCacheKey, FSDataInputStream>() {
@Override
- public FSDataInputStream load(DFSInputStreamCaheKey key) throws Exception {
- DFSClient client = getDfsClient(key.userId);
+ public FSDataInputStream
+ load(DFSInputStreamCacheKey key) throws Exception {
+ DFSClient client = getDfsClient(key.userId, key.namenodeId);
DFSInputStream dis = client.open(key.inodePath);
return client.createWrappedInputStream(dis);
}
};
}
- DFSClient getDfsClient(String userName) {
+ DFSClient getDfsClient(String userName, int namenodeId) {
DFSClient client = null;
try {
- client = clientCache.get(userName);
+ client = clientCache.get(new DfsClientKey(userName, namenodeId));
} catch (ExecutionException e) {
LOG.error("Failed to create DFSClient for user:" + userName + " Cause:"
+ e);
@@ -257,8 +335,10 @@ class DFSClientCache {
return client;
}
- FSDataInputStream getDfsInputStream(String userName, String inodePath) {
- DFSInputStreamCaheKey k = new DFSInputStreamCaheKey(userName, inodePath);
+ FSDataInputStream getDfsInputStream(String userName, String inodePath,
+ int namenodeId) {
+ DFSInputStreamCacheKey k =
+ new DFSInputStreamCacheKey(userName, inodePath, namenodeId);
FSDataInputStream s = null;
try {
s = inputstreamCache.get(k);
@@ -269,8 +349,10 @@ class DFSClientCache {
return s;
}
- public void invalidateDfsInputStream(String userName, String inodePath) {
- DFSInputStreamCaheKey k = new DFSInputStreamCaheKey(userName, inodePath);
+ public void invalidateDfsInputStream(String userName, String inodePath,
+ int namenodeId) {
+ DFSInputStreamCacheKey k =
+ new DFSInputStreamCacheKey(userName, inodePath, namenodeId);
inputstreamCache.invalidate(k);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
index e376ebd..c6da198 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
@@ -18,8 +18,17 @@
package org.apache.hadoop.hdfs.nfs.nfs3;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.file.FileSystemException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.nfs.NfsFileType;
import org.apache.hadoop.nfs.NfsTime;
@@ -224,4 +233,41 @@ public class Nfs3Utils {
public static long getElapsedTime(long startTimeNano) {
return System.nanoTime() - startTimeNano;
}
+
+ public static int getNamenodeId(Configuration conf) {
+ URI filesystemURI = FileSystem.getDefaultUri(conf);
+ return getNamenodeId(conf, filesystemURI);
+ }
+
+ public static int getNamenodeId(Configuration conf, URI namenodeURI) {
+ InetSocketAddress address =
+ DFSUtilClient.getNNAddressCheckLogical(conf, namenodeURI);
+ return address.hashCode();
+ }
+
+ public static URI getResolvedURI(FileSystem fs, String exportPath)
+ throws IOException {
+ URI fsURI = fs.getUri();
+ String scheme = fs.getScheme();
+ if (scheme.equalsIgnoreCase(FsConstants.VIEWFS_SCHEME)) {
+ ViewFileSystem viewFs = (ViewFileSystem)fs;
+ ViewFileSystem.MountPoint[] mountPoints = viewFs.getMountPoints();
+ for (ViewFileSystem.MountPoint mount : mountPoints) {
+ String mountedPath = mount.getMountedOnPath().toString();
+ if (exportPath.startsWith(mountedPath)) {
+ String subpath = exportPath.substring(mountedPath.length());
+ fsURI = mount.getTargetFileSystemURIs()[0].resolve(subpath);
+ break;
+ }
+ }
+ } else if (scheme.equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) {
+ fsURI = fsURI.resolve(exportPath);
+ }
+
+ if (!fsURI.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) {
+ throw new FileSystemException("Only HDFS is supported as underlying"
+ + "FileSystem, fs scheme:" + scheme + " uri to be added" + fsURI);
+ }
+ return fsURI;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
index 2617019..5b7dc14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
@@ -442,7 +442,7 @@ class OpenFileCtx {
if (!activeState) {
LOG.info("OpenFileCtx is inactive, fileId: "
- + request.getHandle().getFileId());
+ + request.getHandle().dumpFileHandle());
WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
fileWcc, 0, request.getStableHow(), Nfs3Constant.WRITE_COMMIT_VERF);
@@ -981,7 +981,8 @@ class OpenFileCtx {
* Check stream status to decide if it should be closed
* @return true, remove stream; false, keep stream
*/
- public synchronized boolean streamCleanup(long fileId, long streamTimeout) {
+ public synchronized boolean streamCleanup(FileHandle handle,
+ long streamTimeout) {
Preconditions
.checkState(streamTimeout >= NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_MIN_DEFAULT);
if (!activeState) {
@@ -992,7 +993,8 @@ class OpenFileCtx {
// Check the stream timeout
if (checkStreamTimeout(streamTimeout)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("stream can be closed for fileId: " + fileId);
+ LOG.debug("stream can be closed for fileId: "
+ + handle.dumpFileHandle());
}
flag = true;
}
@@ -1188,7 +1190,7 @@ class OpenFileCtx {
FileHandle handle = writeCtx.getHandle();
if (LOG.isDebugEnabled()) {
- LOG.debug("do write, fileId: " + handle.getFileId() + " offset: "
+ LOG.debug("do write, fileHandle " + handle.dumpFileHandle() + " offset: "
+ offset + " length: " + count + " stableHow: " + stableHow.name());
}
@@ -1213,8 +1215,9 @@ class OpenFileCtx {
writeCtx.setDataState(WriteCtx.DataState.NO_DUMP);
updateNonSequentialWriteInMemory(-count);
if (LOG.isDebugEnabled()) {
- LOG.debug("After writing " + handle.getFileId() + " at offset "
- + offset + ", updated the memory count, new value: "
+ LOG.debug("After writing " + handle.dumpFileHandle()
+ + " at offset " + offset
+ + ", updated the memory count, new value: "
+ nonSequentialWriteInMemory.get());
}
}
@@ -1257,8 +1260,8 @@ class OpenFileCtx {
processCommits(writeCtx.getOffset() + writeCtx.getCount());
} catch (IOException e) {
- LOG.error("Error writing to fileId " + handle.getFileId() + " at offset "
- + offset + " and length " + count, e);
+ LOG.error("Error writing to fileHandle " + handle.dumpFileHandle()
+ + " at offset " + offset + " and length " + count, e);
if (!writeCtx.getReplied()) {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO);
Nfs3Utils.writeChannel(channel, response.serialize(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
index e26fac5..e23e490 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
@@ -156,7 +156,7 @@ class OpenFileCtxCache {
Entry<FileHandle, OpenFileCtx> pairs = it.next();
FileHandle handle = pairs.getKey();
OpenFileCtx ctx = pairs.getValue();
- if (!ctx.streamCleanup(handle.getFileId(), streamTimeout)) {
+ if (!ctx.streamCleanup(handle, streamTimeout)) {
continue;
}
@@ -164,10 +164,10 @@ class OpenFileCtxCache {
synchronized (this) {
OpenFileCtx ctx2 = openFileMap.get(handle);
if (ctx2 != null) {
- if (ctx2.streamCleanup(handle.getFileId(), streamTimeout)) {
+ if (ctx2.streamCleanup(handle, streamTimeout)) {
openFileMap.remove(handle);
if (LOG.isDebugEnabled()) {
- LOG.debug("After remove stream " + handle.getFileId()
+ LOG.debug("After remove stream " + handle.dumpFileHandle()
+ ", the stream number:" + size());
}
ctxToRemove.add(ctx2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
index 7a6aa89..0db633f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
@@ -319,12 +319,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
GETATTR3Request request;
try {
request = GETATTR3Request.deserialize(xdr);
@@ -335,9 +329,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("GETATTR for fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("GETATTR for fileHandle: " + handle.dumpFileHandle()
+ + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
Nfs3FileAttributes attrs = null;
@@ -412,11 +414,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
SETATTR3Response setattr(XDR xdr, SecurityHandler securityHandler,
SocketAddress remoteAddress) {
SETATTR3Response response = new SETATTR3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
SETATTR3Request request;
try {
@@ -428,9 +425,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS SETATTR fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("NFS SETATTR fileHandle: " + handle.dumpFileHandle()
+ + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
if (request.getAttr().getUpdateFields().contains(SetAttrField.SIZE)) {
@@ -498,12 +503,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
LOOKUP3Request request;
try {
request = LOOKUP3Request.deserialize(xdr);
@@ -514,15 +513,22 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
FileHandle dirHandle = request.getHandle();
String fileName = request.getName();
+ int namenodeId = dirHandle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS LOOKUP dir fileId: " + dirHandle.getFileId() + " name: "
- + fileName + " client: " + remoteAddress);
+ LOG.debug("NFS LOOKUP dir fileHandle: " + dirHandle.dumpFileHandle()
+ + " name: " + fileName + " client: " + remoteAddress);
}
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
try {
String dirFileIdPath = Nfs3Utils.getFileIdPath(dirHandle);
Nfs3FileAttributes postOpObjAttr = writeManager.getFileAttr(dfsClient,
- dirHandle, fileName);
+ dirHandle, fileName, namenodeId);
if (postOpObjAttr == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("NFS LOOKUP fileId: " + dirHandle.getFileId() + " name: "
@@ -540,7 +546,8 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
LOG.info("Can't get path for dir fileId: " + dirHandle.getFileId());
return new LOOKUP3Response(Nfs3Status.NFS3ERR_STALE);
}
- FileHandle fileHandle = new FileHandle(postOpObjAttr.getFileId());
+ FileHandle fileHandle =
+ new FileHandle(postOpObjAttr.getFileId(), namenodeId);
return new LOOKUP3Response(Nfs3Status.NFS3_OK, fileHandle, postOpObjAttr,
postOpDirAttr);
@@ -566,12 +573,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
ACCESS3Request request;
try {
request = ACCESS3Request.deserialize(xdr);
@@ -581,13 +582,21 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
- Nfs3FileAttributes attrs;
+ int namenodeId = handle.getNamenodeId();
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS ACCESS fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("NFS ACCESS fileHandle: " + handle.dumpFileHandle()
+ + " client: " + remoteAddress);
}
+ Nfs3FileAttributes attrs;
try {
attrs = writeManager.getFileAttr(dfsClient, handle, iug);
@@ -639,12 +648,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
READLINK3Request request;
try {
@@ -655,9 +658,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS READLINK fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("NFS READLINK fileHandle: " + handle.dumpFileHandle()
+ + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
String fileIdPath = Nfs3Utils.getFileIdPath(handle);
@@ -715,12 +726,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(userName);
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
READ3Request request;
try {
@@ -734,9 +739,16 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
int count = request.getCount();
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS READ fileId: " + handle.getFileId() + " offset: " + offset
- + " count: " + count + " client: " + remoteAddress);
+ LOG.debug("NFS READ fileHandle: " + handle.dumpFileHandle()+ " offset: "
+ + offset + " count: " + count + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient = clientCache.getDfsClient(userName, namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
Nfs3FileAttributes attrs;
@@ -791,7 +803,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
*/
for (int i = 0; i < 1; ++i) {
FSDataInputStream fis = clientCache.getDfsInputStream(userName,
- Nfs3Utils.getFileIdPath(handle));
+ Nfs3Utils.getFileIdPath(handle), namenodeId);
if (fis == null) {
return new READ3Response(Nfs3Status.NFS3ERR_ACCES);
@@ -805,7 +817,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
// which requires incompatible changes.
if (e.getMessage().equals("Stream closed")) {
clientCache.invalidateDfsInputStream(userName,
- Nfs3Utils.getFileIdPath(handle));
+ Nfs3Utils.getFileIdPath(handle), namenodeId);
continue;
} else {
throw e;
@@ -850,11 +862,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
SecurityHandler securityHandler, SocketAddress remoteAddress) {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
WRITE3Request request;
@@ -875,12 +882,20 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS WRITE fileId: " + handle.getFileId() + " offset: "
+ LOG.debug("NFS WRITE fileHandle: " + handle.dumpFileHandle() + " offset: "
+ offset + " length: " + count + " stableHow: " + stableHow.getValue()
+ " xid: " + xid + " client: " + remoteAddress);
}
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
Nfs3FileAttributes preOpAttr = null;
try {
preOpAttr = writeManager.getFileAttr(dfsClient, handle, iug);
@@ -932,11 +947,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
CREATE3Response create(XDR xdr, SecurityHandler securityHandler,
SocketAddress remoteAddress) {
CREATE3Response response = new CREATE3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
CREATE3Request request;
@@ -949,11 +959,19 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
FileHandle dirHandle = request.getHandle();
String fileName = request.getName();
+ int namenodeId = dirHandle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS CREATE dir fileId: " + dirHandle.getFileId()
+ LOG.debug("NFS CREATE dir fileHandle: " + dirHandle.dumpFileHandle()
+ " filename: " + fileName + " client: " + remoteAddress);
}
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
int createMode = request.getMode();
if ((createMode != Nfs3Constant.CREATE_EXCLUSIVE)
&& request.getObjAttr().getUpdateFields().contains(SetAttrField.SIZE)
@@ -1016,7 +1034,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
OpenFileCtx openFileCtx = new OpenFileCtx(fos, postOpObjAttr,
writeDumpDir + "/" + postOpObjAttr.getFileId(), dfsClient, iug,
aixCompatMode, config);
- fileHandle = new FileHandle(postOpObjAttr.getFileId());
+ fileHandle = new FileHandle(postOpObjAttr.getFileId(), namenodeId);
if (!writeManager.addOpenFileStream(fileHandle, openFileCtx)) {
LOG.warn("Can't add more stream, close it."
+ " Future write will become append");
@@ -1066,11 +1084,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
MKDIR3Response mkdir(XDR xdr, SecurityHandler securityHandler,
SocketAddress remoteAddress) {
MKDIR3Response response = new MKDIR3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
MKDIR3Request request;
@@ -1082,9 +1095,18 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle dirHandle = request.getHandle();
String fileName = request.getName();
+ int namenodeId = dirHandle.getNamenodeId();
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS MKDIR dirId: " + dirHandle.getFileId() + " filename: "
- + fileName + " client: " + remoteAddress);
+ LOG.debug("NFS MKDIR dirHandle: " + dirHandle.dumpFileHandle()
+ + " filename: " + fileName + " client: " + remoteAddress);
}
if (request.getObjAttr().getUpdateFields().contains(SetAttrField.SIZE)) {
@@ -1130,11 +1152,11 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
setattrInternal(dfsClient, fileIdPath, setAttr3, false);
postOpObjAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
- objFileHandle = new FileHandle(postOpObjAttr.getFileId());
+ objFileHandle = new FileHandle(postOpObjAttr.getFileId(), namenodeId);
WccData dirWcc = Nfs3Utils.createWccData(
Nfs3Utils.getWccAttr(preOpDirAttr), dfsClient, dirFileIdPath, iug);
return new MKDIR3Response(Nfs3Status.NFS3_OK, new FileHandle(
- postOpObjAttr.getFileId()), postOpObjAttr, dirWcc);
+ postOpObjAttr.getFileId(), namenodeId), postOpObjAttr, dirWcc);
} catch (IOException e) {
LOG.warn("Exception ", e);
// Try to return correct WccData
@@ -1167,11 +1189,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
REMOVE3Response remove(XDR xdr, SecurityHandler securityHandler,
SocketAddress remoteAddress) {
REMOVE3Response response = new REMOVE3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
REMOVE3Request request;
try {
@@ -1181,12 +1198,21 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return new REMOVE3Response(Nfs3Status.NFS3ERR_INVAL);
}
FileHandle dirHandle = request.getHandle();
+ int namenodeId = dirHandle.getNamenodeId();
+
String fileName = request.getName();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS REMOVE dir fileId: " + dirHandle.getFileId()
+ LOG.debug("NFS REMOVE dir fileHandle: " + dirHandle.dumpFileHandle()
+ " fileName: " + fileName + " client: " + remoteAddress);
}
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
String dirFileIdPath = Nfs3Utils.getFileIdPath(dirHandle);
Nfs3FileAttributes preOpDirAttr = null;
Nfs3FileAttributes postOpDirAttr = null;
@@ -1247,11 +1273,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
RMDIR3Response rmdir(XDR xdr, SecurityHandler securityHandler,
SocketAddress remoteAddress) {
RMDIR3Response response = new RMDIR3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
RMDIR3Request request;
try {
@@ -1262,12 +1283,19 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle dirHandle = request.getHandle();
String fileName = request.getName();
-
+ int namenodeId = dirHandle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS RMDIR dir fileId: " + dirHandle.getFileId()
+ LOG.debug("NFS RMDIR dir fileHandle: " + dirHandle.dumpFileHandle()
+ " fileName: " + fileName + " client: " + remoteAddress);
}
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
String dirFileIdPath = Nfs3Utils.getFileIdPath(dirHandle);
Nfs3FileAttributes preOpDirAttr = null;
Nfs3FileAttributes postOpDirAttr = null;
@@ -1332,11 +1360,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
RENAME3Response rename(XDR xdr, SecurityHandler securityHandler,
SocketAddress remoteAddress) {
RENAME3Response response = new RENAME3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
RENAME3Request request = null;
try {
@@ -1347,13 +1370,28 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle fromHandle = request.getFromDirHandle();
+ int fromNamenodeId = fromHandle.getNamenodeId();
String fromName = request.getFromName();
FileHandle toHandle = request.getToDirHandle();
+ int toNamenodeId = toHandle.getNamenodeId();
String toName = request.getToName();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS RENAME from: " + fromHandle.getFileId() + "/" + fromName
- + " to: " + toHandle.getFileId() + "/" + toName + " client: "
- + remoteAddress);
+ LOG.debug("NFS RENAME from: " + fromHandle.dumpFileHandle()
+ + "/" + fromName + " to: " + toHandle.dumpFileHandle()
+ + "/" + toName + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), fromNamenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
+ if (fromNamenodeId != toNamenodeId) {
+ // renaming file from one namenode to another is not supported
+ response.setStatus(Nfs3Status.NFS3ERR_INVAL);
+ return response;
}
String fromDirFileIdPath = Nfs3Utils.getFileIdPath(fromHandle);
@@ -1429,12 +1467,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
SYMLINK3Request request;
try {
request = SYMLINK3Request.deserialize(xdr);
@@ -1448,11 +1480,20 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
String name = request.getName();
String symData = request.getSymData();
String linkDirIdPath = Nfs3Utils.getFileIdPath(dirHandle);
+ int namenodeId = dirHandle.getNamenodeId();
+
// Don't do any name check to source path, just leave it to HDFS
String linkIdPath = linkDirIdPath + "/" + name;
if (LOG.isDebugEnabled()) {
LOG.debug("NFS SYMLINK, target: " + symData + " link: " + linkIdPath
- + " client: " + remoteAddress);
+ + " namenodeId: " + namenodeId + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
try {
@@ -1471,7 +1512,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
.setPostOpAttr(Nfs3Utils.getFileAttr(dfsClient, linkDirIdPath, iug));
return new SYMLINK3Response(Nfs3Status.NFS3_OK, new FileHandle(
- objAttr.getFileId()), objAttr, dirWcc);
+ objAttr.getFileId(), namenodeId), objAttr, dirWcc);
} catch (IOException e) {
LOG.warn("Exception: " + e);
@@ -1524,12 +1565,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
READDIR3Request request;
try {
request = READDIR3Request.deserialize(xdr);
@@ -1538,6 +1573,8 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return new READDIR3Response(Nfs3Status.NFS3ERR_INVAL);
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
+
long cookie = request.getCookie();
if (cookie < 0) {
LOG.error("Invalid READDIR request, with negative cookie: " + cookie);
@@ -1550,8 +1587,16 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS READDIR fileId: " + handle.getFileId() + " cookie: "
- + cookie + " count: " + count + " client: " + remoteAddress);
+ LOG.debug("NFS READDIR fileHandle: " + handle.dumpFileHandle()
+ + " cookie: " + cookie + " count: " + count + " client: "
+ + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
HdfsFileStatus dirStatus;
@@ -1684,10 +1729,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_ACCES);
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_SERVERFAULT);
- }
READDIRPLUS3Request request = null;
try {
@@ -1698,6 +1739,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
long cookie = request.getCookie();
if (cookie < 0) {
LOG.error("Invalid READDIRPLUS request, with negative cookie: " + cookie);
@@ -1715,9 +1757,15 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS READDIRPLUS fileId: " + handle.getFileId() + " cookie: "
- + cookie + " dirCount: " + dirCount + " maxCount: " + maxCount
- + " client: " + remoteAddress);
+ LOG.debug("NFS READDIRPLUS fileHandle: " + handle.dumpFileHandle()
+ + " cookie: " + cookie + " dirCount: " + dirCount + " maxCount: "
+ + maxCount + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_SERVERFAULT);
}
HdfsFileStatus dirStatus;
@@ -1805,14 +1853,14 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
entries[0] = new READDIRPLUS3Response.EntryPlus3(
postOpDirAttr.getFileId(), ".", 0, postOpDirAttr, new FileHandle(
- postOpDirAttr.getFileId()));
+ postOpDirAttr.getFileId(), namenodeId));
entries[1] = new READDIRPLUS3Response.EntryPlus3(dotdotFileId, "..",
dotdotFileId, Nfs3Utils.getNfs3FileAttrFromFileStatus(dotdotStatus,
- iug), new FileHandle(dotdotFileId));
+ iug), new FileHandle(dotdotFileId, namenodeId));
for (int i = 2; i < n + 2; i++) {
long fileId = fstatus[i - 2].getFileId();
- FileHandle childHandle = new FileHandle(fileId);
+ FileHandle childHandle = new FileHandle(fileId, namenodeId);
Nfs3FileAttributes attr;
try {
attr = writeManager.getFileAttr(dfsClient, childHandle, iug);
@@ -1829,7 +1877,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
entries = new READDIRPLUS3Response.EntryPlus3[n];
for (int i = 0; i < n; i++) {
long fileId = fstatus[i].getFileId();
- FileHandle childHandle = new FileHandle(fileId);
+ FileHandle childHandle = new FileHandle(fileId, namenodeId);
Nfs3FileAttributes attr;
try {
attr = writeManager.getFileAttr(dfsClient, childHandle, iug);
@@ -1863,11 +1911,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
FSSTAT3Request request;
try {
@@ -1878,9 +1921,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS FSSTAT fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("NFS FSSTAT fileHandle: " + handle.dumpFileHandle()
+ + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
try {
@@ -1938,12 +1989,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
FSINFO3Request request;
try {
request = FSINFO3Request.deserialize(xdr);
@@ -1953,9 +1998,17 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS FSINFO fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("NFS FSINFO fileHandle: " + handle.dumpFileHandle()
+ +" client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
try {
@@ -2003,12 +2056,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
return response;
}
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
-
PATHCONF3Request request;
try {
request = PATHCONF3Request.deserialize(xdr);
@@ -2019,10 +2066,18 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
FileHandle handle = request.getHandle();
Nfs3FileAttributes attrs;
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS PATHCONF fileId: " + handle.getFileId() + " client: "
- + remoteAddress);
+ LOG.debug("NFS PATHCONF fileHandle: " + handle.dumpFileHandle()
+ + " client: " + remoteAddress);
+ }
+
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
}
try {
@@ -2055,11 +2110,6 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
COMMIT3Response commit(XDR xdr, Channel channel, int xid,
SecurityHandler securityHandler, SocketAddress remoteAddress) {
COMMIT3Response response = new COMMIT3Response(Nfs3Status.NFS3_OK);
- DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
- if (dfsClient == null) {
- response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
- return response;
- }
COMMIT3Request request;
try {
@@ -2071,12 +2121,20 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
}
FileHandle handle = request.getHandle();
+ int namenodeId = handle.getNamenodeId();
if (LOG.isDebugEnabled()) {
- LOG.debug("NFS COMMIT fileId: " + handle.getFileId() + " offset="
+ LOG.debug("NFS COMMIT fileHandle: " + handle.dumpFileHandle() + " offset="
+ request.getOffset() + " count=" + request.getCount() + " client: "
+ remoteAddress);
}
+ DFSClient dfsClient =
+ clientCache.getDfsClient(securityHandler.getUser(), namenodeId);
+ if (dfsClient == null) {
+ response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
+ return response;
+ }
+
String fileIdPath = Nfs3Utils.getFileIdPath(handle);
Nfs3FileAttributes preOpAttr = null;
try {
@@ -2097,7 +2155,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
// Insert commit as an async request
writeManager.handleCommit(dfsClient, handle, commitOffset, channel, xid,
- preOpAttr);
+ preOpAttr, namenodeId);
return null;
} catch (IOException e) {
LOG.warn("Exception ", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
index f89679f..5d66751 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
@@ -318,9 +318,9 @@ class WriteCtx {
@Override
public String toString() {
- return "Id:" + handle.getFileId() + " offset:" + getPlainOffset() + " " +
- "count:" + count + " originalCount:" + getOriginalCount() +
- " stableHow:" + stableHow + " replied:" + replied + " dataState:" +
- dataState + " xid:" + xid;
+ return "FileHandle:" + handle.dumpFileHandle() + " offset:"
+ + getPlainOffset() + " " + "count:" + count + " originalCount:"
+ + getOriginalCount() + " stableHow:" + stableHow + " replied:"
+ + replied + " dataState:" + dataState + " xid:" + xid;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
index 7810ce2..0a3450d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
@@ -139,7 +139,8 @@ public class WriteManager {
FileHandle fileHandle = request.getHandle();
OpenFileCtx openFileCtx = fileContextCache.get(fileHandle);
if (openFileCtx == null) {
- LOG.info("No opened stream for fileId: " + fileHandle.getFileId());
+ LOG.info("No opened stream for fileHandle: "
+ + fileHandle.dumpFileHandle());
String fileIdPath = Nfs3Utils.getFileIdPath(fileHandle.getFileId());
HdfsDataOutputStream fos = null;
@@ -188,7 +189,8 @@ public class WriteManager {
try {
fos.close();
} catch (IOException e) {
- LOG.error("Can't close stream for fileId: " + handle.getFileId(), e);
+ LOG.error("Can't close stream for fileHandle: "
+ + handle.dumpFileHandle(), e);
}
// Notify client to retry
WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
@@ -201,7 +203,8 @@ public class WriteManager {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Opened stream for appending file: " + fileHandle.getFileId());
+ LOG.debug("Opened stream for appending file: "
+ + fileHandle.dumpFileHandle());
}
}
@@ -220,7 +223,7 @@ public class WriteManager {
if (openFileCtx == null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("No opened stream for fileId: " + fileHandle.getFileId()
+ LOG.debug("No opened stream for fileId: " + fileHandle.dumpFileHandle()
+ " commitOffset=" + commitOffset
+ ". Return success in this case.");
}
@@ -263,13 +266,14 @@ public class WriteManager {
}
void handleCommit(DFSClient dfsClient, FileHandle fileHandle,
- long commitOffset, Channel channel, int xid, Nfs3FileAttributes preOpAttr) {
+ long commitOffset, Channel channel, int xid, Nfs3FileAttributes preOpAttr,
+ int namenodeId) {
long startTime = System.nanoTime();
int status;
OpenFileCtx openFileCtx = fileContextCache.get(fileHandle);
if (openFileCtx == null) {
- LOG.info("No opened stream for fileId: " + fileHandle.getFileId()
+ LOG.info("No opened stream for fileId: " + fileHandle.dumpFileHandle()
+ " commitOffset=" + commitOffset + ". Return success in this case.");
status = Nfs3Status.NFS3_OK;
@@ -304,7 +308,9 @@ public class WriteManager {
// Send out the response
Nfs3FileAttributes postOpAttr = null;
try {
- postOpAttr = getFileAttr(dfsClient, new FileHandle(preOpAttr.getFileId()), iug);
+ postOpAttr =
+ getFileAttr(dfsClient, new FileHandle(preOpAttr.getFileId(),
+ namenodeId), iug);
} catch (IOException e1) {
LOG.info("Can't get postOpAttr for fileId: " + preOpAttr.getFileId(), e1);
}
@@ -334,13 +340,13 @@ public class WriteManager {
}
Nfs3FileAttributes getFileAttr(DFSClient client, FileHandle dirHandle,
- String fileName) throws IOException {
+ String fileName, int namenodeId) throws IOException {
String fileIdPath = Nfs3Utils.getFileIdPath(dirHandle) + "/" + fileName;
Nfs3FileAttributes attr = Nfs3Utils.getFileAttr(client, fileIdPath, iug);
if ((attr != null) && (attr.getType() == NfsFileType.NFSREG.toValue())) {
OpenFileCtx openFileCtx = fileContextCache.get(new FileHandle(attr
- .getFileId()));
+ .getFileId(), namenodeId));
if (openFileCtx != null) {
attr.setSize(openFileCtx.getNextOffset());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestClientAccessPrivilege.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestClientAccessPrivilege.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestClientAccessPrivilege.java
index b68fdb8..007803d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestClientAccessPrivilege.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestClientAccessPrivilege.java
@@ -101,9 +101,10 @@ public class TestClientAccessPrivilege {
// Create a remove request
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
handle.serialize(xdr_req);
xdr_req.writeString("f1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
index 54b7ee7..ba9d46e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
@@ -43,15 +43,17 @@ public class TestDFSClientCache {
DFSClientCache cache = new DFSClientCache(conf, MAX_CACHE_SIZE);
- DFSClient c1 = cache.getDfsClient("test1");
- assertTrue(cache.getDfsClient("test1").toString().contains("ugi=test1"));
- assertEquals(c1, cache.getDfsClient("test1"));
+ int namenodeId = Nfs3Utils.getNamenodeId(conf);
+ DFSClient c1 = cache.getDfsClient("test1", namenodeId);
+ assertTrue(cache.getDfsClient("test1", namenodeId)
+ .toString().contains("ugi=test1"));
+ assertEquals(c1, cache.getDfsClient("test1", namenodeId));
assertFalse(isDfsClientClose(c1));
- cache.getDfsClient("test2");
+ cache.getDfsClient("test2", namenodeId);
assertTrue(isDfsClientClose(c1));
assertTrue("cache size should be the max size or less",
- cache.clientCache.size() <= MAX_CACHE_SIZE);
+ cache.getClientCache().size() <= MAX_CACHE_SIZE);
}
@Test
@@ -61,6 +63,7 @@ public class TestDFSClientCache {
NfsConfiguration conf = new NfsConfiguration();
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost");
UserGroupInformation currentUserUgi
= UserGroupInformation.createRemoteUser(currentUser);
currentUserUgi.setAuthenticationMethod(KERBEROS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
index c802be0..211a166 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
@@ -21,7 +21,14 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ConfigUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.mount.Mountd;
@@ -31,7 +38,152 @@ import org.junit.Test;
public class TestExportsTable {
@Test
- public void testExportPoint() throws IOException {
+ public void testHdfsExportPoint() throws IOException {
+ NfsConfiguration config = new NfsConfiguration();
+ MiniDFSCluster cluster = null;
+
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+ config.set("nfs.http.address", "0.0.0.0:0");
+
+ try {
+ cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+ cluster.waitActive();
+
+ // Start nfs
+ final Nfs3 nfsServer = new Nfs3(config);
+ nfsServer.startServiceInternal(false);
+
+ Mountd mountd = nfsServer.getMountd();
+ RpcProgramMountd rpcMount = (RpcProgramMountd) mountd.getRpcProgram();
+ assertTrue(rpcMount.getExports().size() == 1);
+
+ String exportInMountd = rpcMount.getExports().get(0);
+ assertTrue(exportInMountd.equals("/"));
+
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testViewFsExportPoint() throws IOException {
+ NfsConfiguration config = new NfsConfiguration();
+ MiniDFSCluster cluster = null;
+ String clusterName = RandomStringUtils.randomAlphabetic(10);
+
+ String exportPoint = "/hdfs1,/hdfs2";
+ config.setStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY, exportPoint);
+ config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.VIEWFS_SCHEME + "://" + clusterName);
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+ config.set("nfs.http.address", "0.0.0.0:0");
+
+ try {
+ cluster =
+ new MiniDFSCluster.Builder(config).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem hdfs1 = cluster.getFileSystem(0);
+ DistributedFileSystem hdfs2 = cluster.getFileSystem(1);
+ cluster.waitActive();
+ Path base1 = new Path("/user1");
+ Path base2 = new Path("/user2");
+ hdfs1.delete(base1, true);
+ hdfs2.delete(base2, true);
+ hdfs1.mkdirs(base1);
+ hdfs2.mkdirs(base2);
+ ConfigUtil.addLink(config, clusterName, "/hdfs1",
+ hdfs1.makeQualified(base1).toUri());
+ ConfigUtil.addLink(config, clusterName, "/hdfs2",
+ hdfs2.makeQualified(base2).toUri());
+
+ // Start nfs
+ final Nfs3 nfsServer = new Nfs3(config);
+ nfsServer.startServiceInternal(false);
+
+ Mountd mountd = nfsServer.getMountd();
+ RpcProgramMountd rpcMount = (RpcProgramMountd) mountd.getRpcProgram();
+ assertTrue(rpcMount.getExports().size() == 2);
+
+ String exportInMountd1 = rpcMount.getExports().get(0);
+ assertTrue(exportInMountd1.equals("/hdfs1"));
+
+ String exportInMountd2 = rpcMount.getExports().get(1);
+ assertTrue(exportInMountd2.equals("/hdfs2"));
+
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testViewFsInternalExportPoint() throws IOException {
+ NfsConfiguration config = new NfsConfiguration();
+ MiniDFSCluster cluster = null;
+ String clusterName = RandomStringUtils.randomAlphabetic(10);
+
+ String exportPoint = "/hdfs1/subpath";
+ config.setStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY, exportPoint);
+ config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.VIEWFS_SCHEME + "://" + clusterName);
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+ config.set("nfs.http.address", "0.0.0.0:0");
+
+ try {
+ cluster =
+ new MiniDFSCluster.Builder(config).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem hdfs1 = cluster.getFileSystem(0);
+ DistributedFileSystem hdfs2 = cluster.getFileSystem(1);
+ cluster.waitActive();
+ Path base1 = new Path("/user1");
+ Path base2 = new Path("/user2");
+ hdfs1.delete(base1, true);
+ hdfs2.delete(base2, true);
+ hdfs1.mkdirs(base1);
+ hdfs2.mkdirs(base2);
+ ConfigUtil.addLink(config, clusterName, "/hdfs1",
+ hdfs1.makeQualified(base1).toUri());
+ ConfigUtil.addLink(config, clusterName, "/hdfs2",
+ hdfs2.makeQualified(base2).toUri());
+ Path subPath = new Path(base1, "subpath");
+ hdfs1.delete(subPath, true);
+ hdfs1.mkdirs(subPath);
+
+ // Start nfs
+ final Nfs3 nfsServer = new Nfs3(config);
+ nfsServer.startServiceInternal(false);
+
+ Mountd mountd = nfsServer.getMountd();
+ RpcProgramMountd rpcMount = (RpcProgramMountd) mountd.getRpcProgram();
+ assertTrue(rpcMount.getExports().size() == 1);
+
+ String exportInMountd = rpcMount.getExports().get(0);
+ assertTrue(exportInMountd.equals(exportPoint));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testHdfsInternalExportPoint() throws IOException {
NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = null;
@@ -40,10 +192,15 @@ public class TestExportsTable {
// Use emphral port in case tests are running in parallel
config.setInt("nfs3.mountd.port", 0);
config.setInt("nfs3.server.port", 0);
-
+ config.set("nfs.http.address", "0.0.0.0:0");
+ Path base = new Path(exportPoint);
+
try {
cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
cluster.waitActive();
+ DistributedFileSystem hdfs = cluster.getFileSystem(0);
+ hdfs.delete(base, true);
+ hdfs.mkdirs(base);
// Start nfs
final Nfs3 nfsServer = new Nfs3(config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
index 05ba2db..0af7ced 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
@@ -21,9 +21,7 @@ package org.apache.hadoop.hdfs.nfs.nfs3;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.SocketAddress;
import java.util.List;
import org.apache.hadoop.fs.Path;
@@ -31,8 +29,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
-import org.apache.hadoop.hdfs.nfs.nfs3.Nfs3;
-import org.apache.hadoop.hdfs.nfs.nfs3.RpcProgramNfs3;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.nfs.nfs3.FileHandle;
@@ -40,15 +36,10 @@ import org.apache.hadoop.nfs.nfs3.response.READDIR3Response;
import org.apache.hadoop.nfs.nfs3.response.READDIR3Response.Entry3;
import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response;
import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response.EntryPlus3;
-import org.apache.hadoop.oncrpc.RpcInfo;
-import org.apache.hadoop.oncrpc.RpcMessage;
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.security.SecurityHandler;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelHandlerContext;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -119,10 +110,11 @@ public class TestReaddir {
// Get inodeId of /tmp
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
// Create related part of the XDR request
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
handle.serialize(xdr_req);
xdr_req.writeLongAsHyper(0); // cookie
xdr_req.writeLongAsHyper(0); // verifier
@@ -139,7 +131,7 @@ public class TestReaddir {
// Create related part of the XDR request
xdr_req = new XDR();
- handle = new FileHandle(dirId);
+ handle = new FileHandle(dirId, namenodeId);
handle.serialize(xdr_req);
xdr_req.writeLongAsHyper(f2Id); // cookie
xdr_req.writeLongAsHyper(0); // verifier
@@ -167,10 +159,11 @@ public class TestReaddir {
// Get inodeId of /tmp
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
// Create related part of the XDR request
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
handle.serialize(xdr_req);
xdr_req.writeLongAsHyper(0); // cookie
xdr_req.writeLongAsHyper(0); // verifier
@@ -189,7 +182,7 @@ public class TestReaddir {
// Create related part of the XDR request
xdr_req = new XDR();
- handle = new FileHandle(dirId);
+ handle = new FileHandle(dirId, namenodeId);
handle.serialize(xdr_req);
xdr_req.writeLongAsHyper(f2Id); // cookie
xdr_req.writeLongAsHyper(0); // verifier
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
index e3c108a..b9d5b82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableTable;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.junit.Assert;
@@ -52,6 +53,8 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
when(q.getQueueCapacities()).thenReturn(qc);
when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i]));
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
list.add(q);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1108f1a..0132348 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -354,10 +354,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
- int numExpectedElements = 18;
+ int numExpectedElements = 20;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 31;
+ numExpectedElements = 33;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-6575. Support global
configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Posted by su...@apache.org.
YARN-6575. Support global configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e566fd8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e566fd8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e566fd8b
Branch: refs/heads/YARN-5881
Commit: e566fd8b58a6570593b21597aa5294d7b37cb388
Parents: a4e6253
Author: Xuan <xg...@apache.org>
Authored: Mon Jun 5 16:30:38 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../ConfigurationMutationACLPolicy.java | 4 +-
.../DefaultConfigurationMutationACLPolicy.java | 4 +-
.../scheduler/MutableConfScheduler.java | 4 +-
.../scheduler/MutableConfigurationProvider.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../conf/MutableCSConfigurationProvider.java | 10 +-
...ueueAdminConfigurationMutationACLPolicy.java | 22 +++-
.../resourcemanager/webapp/RMWebServices.java | 4 +-
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 -----------
.../webapp/dao/SchedConfUpdateInfo.java | 69 +++++++++++++
.../TestConfigurationMutationACLPolicies.java | 28 ++++-
.../TestMutableCSConfigurationProvider.java | 10 +-
.../TestRMWebServicesConfigurationMutation.java | 101 +++++++++++++------
13 files changed, 205 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
index 724487b..3a388fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
/**
* Interface for determining whether configuration mutations are allowed.
@@ -41,7 +41,7 @@ public interface ConfigurationMutationACLPolicy {
* @param confUpdate configurations to be updated
* @return whether provided mutation is allowed or not
*/
- boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ boolean isMutationAllowed(UserGroupInformation user, SchedConfUpdateInfo
confUpdate);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
index 680c3b8..6648668 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
/**
* Default configuration mutation ACL policy. Checks if user is YARN admin.
@@ -39,7 +39,7 @@ public class DefaultConfigurationMutationACLPolicy implements
@Override
public boolean isMutationAllowed(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) {
+ SchedConfUpdateInfo confUpdate) {
return authorizer.isAdmin(user);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 93a935e..027d944 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -36,7 +36,7 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException;
+ SchedConfUpdateInfo confUpdate) throws IOException;
/**
* Get the scheduler configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index f04c128..6b8306c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -34,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 4f89f7e..4d367e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,7 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -2617,7 +2617,7 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
user, confUpdate);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 8b879b0..eb97260 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.ArrayList;
@@ -98,7 +98,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
@Override
public void mutateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
" queues.");
@@ -126,7 +126,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ SchedConfUpdateInfo mutationInfo) throws IOException {
CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
CapacitySchedulerConfiguration proposedConf =
new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
@@ -140,6 +140,10 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
updateQueue(updateQueueInfo, proposedConf, confUpdate);
}
+ for (Map.Entry<String, String> global : mutationInfo.getGlobalParams()
+ .entrySet()) {
+ confUpdate.put(global.getKey(), global.getValue());
+ }
return confUpdate;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
index 1f94c1c..0a82d50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -22,15 +22,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
/**
@@ -40,16 +42,29 @@ import java.util.Set;
public class QueueAdminConfigurationMutationACLPolicy implements
ConfigurationMutationACLPolicy {
+ private Configuration conf;
private RMContext rmContext;
+ private YarnAuthorizationProvider authorizer;
@Override
- public void init(Configuration conf, RMContext context) {
+ public void init(Configuration config, RMContext context) {
+ this.conf = config;
this.rmContext = context;
+ this.authorizer = YarnAuthorizationProvider.getInstance(conf);
}
@Override
public boolean isMutationAllowed(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) {
+ SchedConfUpdateInfo confUpdate) {
+ // If there are global config changes, check if user is admin.
+ Map<String, String> globalParams = confUpdate.getGlobalParams();
+ if (globalParams != null && globalParams.size() != 0) {
+ if (!authorizer.isAdmin(user)) {
+ return false;
+ }
+ }
+
+ // Check if user is admin of all modified queues.
Set<String> queues = new HashSet<>();
for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
queues.add(addQueueInfo.getQueue());
@@ -71,7 +86,6 @@ public class QueueAdminConfigurationMutationACLPolicy implements
// Queue is not found, do nothing.
}
String parentPath = queuePath;
- // TODO: handle global config change.
while (queueInfo == null) {
// We are adding a queue (whose parent we are possibly also adding).
// Check ACL of lowest parent queue which already exists.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index d95465b..b97e212 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2459,11 +2459,11 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
@PUT
- @Path("/queues")
+ @Path("/sched-conf")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ public Response updateSchedulerConfiguration(SchedConfUpdateInfo
mutationInfo, @Context HttpServletRequest hsr)
throws AuthorizationException, InterruptedException {
init();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
deleted file mode 100644
index 644ec90..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
-
-import java.util.ArrayList;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for making scheduler configuration changes (supports adding,
- * removing, or updating a queue).
- */
-@XmlRootElement(name = "schedConf")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class QueueConfigsUpdateInfo {
-
- @XmlElement(name = "add")
- private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "remove")
- private ArrayList<String> removeQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "update")
- private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
-
- public QueueConfigsUpdateInfo() {
- // JAXB needs this
- }
-
- public ArrayList<QueueConfigInfo> getAddQueueInfo() {
- return addQueueInfo;
- }
-
- public ArrayList<String> getRemoveQueueInfo() {
- return removeQueueInfo;
- }
-
- public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
- return updateQueueInfo;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
new file mode 100644
index 0000000..b7c585e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue, as well as global scheduler conf changes).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SchedConfUpdateInfo {
+
+ @XmlElement(name = "add-queue")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove-queue")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update-queue")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ private HashMap<String, String> global = new HashMap<>();
+
+ public SchedConfUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+
+ @XmlElementWrapper(name = "global-updates")
+ public HashMap<String, String> getGlobalParams() {
+ return global;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
index 4016dcf..0f5a3d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -77,6 +77,7 @@ public class TestConfigurationMutationACLPolicies {
.thenReturn(false);
when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
}
+
@Test
public void testDefaultPolicy() {
Configuration conf = new Configuration();
@@ -98,7 +99,7 @@ public class TestConfigurationMutationACLPolicies {
ConfigurationMutationACLPolicy.class);
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
updateInfo.getUpdateQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -114,7 +115,7 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Add root.b.b1. Should check ACL of root.b queue.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
updateInfo.getAddQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -130,7 +131,7 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Add root.b.b1.b11. Should check ACL of root.b queue.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
updateInfo.getAddQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -146,9 +147,26 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Remove root.b.b1.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.b.b1");
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
}
+
+ @Test
+ public void testQueueAdminPolicyGlobal() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertTrue(policy.isMutationAllowed(BAD_USER, updateInfo));
+ updateInfo.getGlobalParams().put("globalKey", "globalValue");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 13229b1..3216781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -47,8 +47,8 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private QueueConfigsUpdateInfo goodUpdate;
- private QueueConfigsUpdateInfo badUpdate;
+ private SchedConfUpdateInfo goodUpdate;
+ private SchedConfUpdateInfo badUpdate;
private CapacityScheduler cs;
private static final UserGroupInformation TEST_USER = UserGroupInformation
@@ -62,14 +62,14 @@ public class TestMutableCSConfigurationProvider {
when(cs.getConfiguration()).thenReturn(
new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new QueueConfigsUpdateInfo();
+ goodUpdate = new SchedConfUpdateInfo();
Map<String, String> goodUpdateMap = new HashMap<>();
goodUpdateMap.put("goodKey", "goodVal");
QueueConfigInfo goodUpdateInfo = new
QueueConfigInfo("root.a", goodUpdateMap);
goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
- badUpdate = new QueueConfigsUpdateInfo();
+ badUpdate = new SchedConfUpdateInfo();
Map<String, String> badUpdateMap = new HashMap<>();
badUpdateMap.put("badKey", "badVal");
QueueConfigInfo badUpdateInfo = new
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e566fd8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index d149055..5fbe36f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -162,7 +162,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Add parent queue root.d with two children d1 and d2.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> d1Capacity = new HashMap<>();
d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
@@ -181,9 +181,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getAddQueueInfo().add(d);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -205,7 +205,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> dCapacity = new HashMap<>();
dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
Map<String, String> bCapacity = new HashMap<>();
@@ -216,9 +216,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -238,13 +238,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.a.a2");
// Remove root.a.a2
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.a.a2");
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -263,13 +263,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.c", "root.c.c1");
// Remove root.c (parent queue)
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.c");
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -288,7 +288,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.a", "root.a.a1", "root.a.a2");
// Remove root.a (parent queue) with capacity 25
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.a");
// Set root.b capacity to 100
@@ -298,9 +298,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -320,7 +320,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.b", "root.c", "root.c.c1");
// Remove root.b and root.c
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.b");
updateInfo.getRemoveQueueInfo().add("root.c");
Map<String, String> aCapacity = new HashMap<>();
@@ -330,9 +330,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(configInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -348,7 +348,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Set state of queues to STOPPED.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> stoppedParam = new HashMap<>();
stoppedParam.put(CapacitySchedulerConfiguration.STATE,
QueueState.STOPPED.toString());
@@ -358,9 +358,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -378,7 +378,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Update config value.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
"0.2");
@@ -393,9 +393,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
0.001f);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -411,9 +411,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -431,7 +431,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Update root.a and root.b capacity to 50.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
@@ -441,9 +441,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -453,6 +453,47 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
}
+ @Test
+ public void testGlobalConfChange() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set maximum-applications to 30000.
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX +
+ "maximum-applications", "30000");
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("sched-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(30000, newCSConf.getMaximumSystemApplications());
+
+ updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX +
+ "maximum-applications", null);
+ // Unset maximum-applications. Should be set to default.
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("sched-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS,
+ newCSConf.getMaximumSystemApplications());
+ }
+
@Override
@After
public void tearDown() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-7237. Cleanup usages of
ResourceProfiles. (Wangda Tan)
Posted by su...@apache.org.
YARN-7237. Cleanup usages of ResourceProfiles. (Wangda Tan)
Change-Id: I959c8531bd48231956068cb7d2312f3235549f9c
(cherry picked from commit 8536db5a7549fa44d9f9e444b41bd5978b5c6ae1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cfbad68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cfbad68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cfbad68
Branch: refs/heads/YARN-5881
Commit: 5cfbad68c0ef6e70d248d4989f9889576d5fcf7e
Parents: 82cd85d
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 9 20:33:44 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Oct 9 20:36:57 2017 -0700
----------------------------------------------------------------------
.../yarn/api/records/ProfileCapability.java | 5 +-
.../src/test/resources/resource-profiles.json | 8 --
.../hadoop/yarn/api/TestProfileCapability.java | 109 ++++++++++++++
.../hadoop/yarn/server/utils/BuilderUtils.java | 1 +
.../server/resourcemanager/ResourceManager.java | 9 +-
.../resource/ResourceProfilesManagerImpl.java | 41 ++----
.../scheduler/AbstractYarnScheduler.java | 48 +------
.../yarn/server/resourcemanager/MockNM.java | 41 ++++--
.../yarn/server/resourcemanager/MockRM.java | 9 ++
.../TestApplicationMasterService.java | 35 -----
.../resource/MockResourceProfileManager.java | 79 +++++++++++
.../resource/TestResourceProfiles.java | 48 ++++++-
...CapacitySchedulerWithMultiResourceTypes.java | 141 +++++++++++++++++++
.../scheduler/capacity/TestUtils.java | 42 ++++--
.../resources/profiles/sample-profiles-1.json | 8 --
.../resources/profiles/sample-profiles-2.json | 8 --
16 files changed, 471 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
index 2cb4670..d6cb635 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
@@ -153,18 +153,17 @@ public abstract class ProfileCapability {
Resource none = Resource.newInstance(0, 0);
Resource resource = Resource.newInstance(0, 0);
String profileName = capability.getProfileName();
- if (profileName.isEmpty()) {
+ if (null == profileName || profileName.isEmpty()) {
profileName = DEFAULT_PROFILE;
}
if (resourceProfilesMap.containsKey(profileName)) {
resource = Resource.newInstance(resourceProfilesMap.get(profileName));
}
-
if (capability.getProfileCapabilityOverride() != null &&
!capability.getProfileCapabilityOverride().equals(none)) {
for (ResourceInformation entry : capability
.getProfileCapabilityOverride().getResources()) {
- if (entry != null && entry.getValue() >= 0) {
+ if (entry != null && entry.getValue() > 0) {
resource.setResourceInformation(entry.getName(), entry);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/resource-profiles.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/resource-profiles.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/resource-profiles.json
index d0f3f72..d1dba9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/resource-profiles.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/resource-profiles.json
@@ -1,16 +1,8 @@
{
- "minimum": {
- "memory-mb" : 1024,
- "vcores" : 1
- },
"default" : {
"memory-mb" : 2048,
"vcores" : 2
},
- "maximum" : {
- "memory-mb": 4096,
- "vcores" : 4
- },
"http" : {
"memory-mb" : 2048,
"vcores" : 2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java
new file mode 100644
index 0000000..cbad3f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test profile capability behavior.
+ */
+public class TestProfileCapability {
+ @Before
+ public void setup() {
+ // Initialize resource map
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
+ riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
+
+ for (int i = 0; i < 5; i++) {
+ String resourceName = "res-" + i;
+ riMap.put(resourceName, ResourceInformation
+ .newInstance(resourceName, "", 0, ResourceTypes.COUNTABLE, 0,
+ Integer.MAX_VALUE));
+ }
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+ }
+
+ @Test
+ public void testConvertProfileCapabilityToResource() {
+ Resource profile1 = Resource.newInstance(1, 1);
+ profile1.setResourceValue("res-0", 1);
+ profile1.setResourceValue("res-1", 1);
+
+ Resource profile2 = Resource.newInstance(2, 2);
+ profile2.setResourceValue("res-0", 2);
+ profile2.setResourceValue("res-1", 2);
+
+ Resource profile3 = Resource.newInstance(3, 3);
+ profile3.setResourceValue("res-0", 3);
+ profile3.setResourceValue("res-1", 3);
+
+ Map<String, Resource> profiles = ImmutableMap.of("profile1", profile1,
+ "profile2", profile2, "profile3", profile3, "default", profile1);
+
+ // Test case 1, set override value to (1, 1, 0), since we only allow
+ // overwrite for positive value, it is still profile1.
+ ProfileCapability pc = ProfileCapability.newInstance("profile1",
+ Resource.newInstance(1, 1));
+ Assert.assertEquals(profile1, ProfileCapability.toResource(pc, profiles));
+
+ // Test case 2, similarly, negative value won't be respected.
+ pc = ProfileCapability.newInstance("profile1",
+ Resource.newInstance(1, -1));
+ Assert.assertEquals(profile1, ProfileCapability.toResource(pc, profiles));
+
+ // Test case 3, do overwrite for memory and vcores, the result is (3,3,1,1)
+ Resource expected = Resource.newInstance(3, 3);
+ expected.setResourceValue("res-0", 1);
+ expected.setResourceValue("res-1", 1);
+ pc = ProfileCapability.newInstance("profile1",
+ Resource.newInstance(3, 3));
+ Assert.assertEquals(expected, ProfileCapability.toResource(pc, profiles));
+
+ // Test case 3, do overwrite for mem and res-1, the result is (3,1,3,1)
+ expected = Resource.newInstance(3, 1);
+ expected.setResourceValue("res-0", 3);
+ expected.setResourceValue("res-1", 1);
+
+ Resource overwrite = Resource.newInstance(3, 0);
+ overwrite.setResourceValue("res-0", 3);
+ overwrite.setResourceValue("res-1", 0);
+
+ pc = ProfileCapability.newInstance("profile1", overwrite);
+ Assert.assertEquals(expected, ProfileCapability.toResource(pc, profiles));
+
+ // Test case 4, when null profile is specified, use default.
+ pc = ProfileCapability.newInstance("", null);
+ Assert.assertEquals(profile1, ProfileCapability.toResource(pc, profiles));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 3b37abd..fec2681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -378,6 +378,7 @@ public class BuilderUtils {
request.setNumContainers(r.getNumContainers());
request.setNodeLabelExpression(r.getNodeLabelExpression());
request.setExecutionTypeRequest(r.getExecutionTypeRequest());
+ request.setProfileCapability(r.getProfileCapability());
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index d9312f2..07f5a76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -227,6 +227,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
return rmDispatcher;
}
+ @VisibleForTesting
+ protected ResourceProfilesManager createResourceProfileManager() {
+ ResourceProfilesManager resourceProfilesManager =
+ new ResourceProfilesManagerImpl();
+ return resourceProfilesManager;
+ }
+
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
@@ -236,7 +243,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
// add resource profiles here because it's used by AbstractYarnScheduler
ResourceProfilesManager resourceProfilesManager =
- new ResourceProfilesManagerImpl();
+ createResourceProfileManager();
resourceProfilesManager.init(conf);
rmContext.setResourceProfilesManager(resourceProfilesManager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index dfd14e0..e980069 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -24,7 +24,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -35,12 +34,9 @@ import org.codehaus.jackson.map.ObjectMapper;
import java.io.File;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -55,8 +51,6 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
LogFactory.getLog(ResourceProfilesManagerImpl.class);
private final Map<String, Resource> profiles = new ConcurrentHashMap<>();
- private List<ResourceTypeInfo> resourceTypeInfo =
- new ArrayList<ResourceTypeInfo>();
private Configuration conf;
private boolean profileEnabled = false;
@@ -87,26 +81,6 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
public void init(Configuration config) throws IOException {
conf = config;
loadProfiles();
-
- // Load resource types, this should be done even if resource profile is
- // disabled, since we have mandatory resource types like vcores/memory.
- loadResourceTypes();
- }
-
- private void loadResourceTypes() {
- // Add all resource types
- try {
- writeLock.lock();
- Collection<ResourceInformation> resourcesInfo = ResourceUtils
- .getResourceTypes().values();
- for (ResourceInformation resourceInfo : resourcesInfo) {
- resourceTypeInfo
- .add(ResourceTypeInfo.newInstance(resourceInfo.getName(),
- resourceInfo.getUnits(), resourceInfo.getResourceType()));
- }
- } finally {
- writeLock.unlock();
- }
}
private void loadProfiles() throws IOException {
@@ -140,6 +114,14 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
throw new IOException(
"Name of resource profile cannot be an empty string");
}
+
+ if (profileName.equals(MINIMUM_PROFILE) || profileName.equals(
+ MAXIMUM_PROFILE)) {
+ throw new IOException(String.format(
+ "profile={%s, %s} is should not be specified "
+ + "inside %s, they will be loaded from resource-types.xml",
+ MINIMUM_PROFILE, MAXIMUM_PROFILE, sourceFile));
+ }
if (entry.getValue() instanceof Map) {
Map profileInfo = (Map) entry.getValue();
// ensure memory and vcores are specified
@@ -155,6 +137,13 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
"Added profile '" + profileName + "' with resources: " + resource);
}
}
+
+ // add minimum/maximum profile
+ profiles.put(MINIMUM_PROFILE,
+ ResourceUtils.getResourceTypesMinimumAllocation());
+ profiles.put(MAXIMUM_PROFILE,
+ ResourceUtils.getResourceTypesMaximumAllocation());
+
// check to make sure mandatory profiles are present
for (String profile : MANDATORY_PROFILES) {
if (!profiles.containsKey(profile)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 9d6037e..f2da1fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
@@ -795,7 +794,7 @@ public abstract class AbstractYarnScheduler
writeLock.unlock();
}
}
-
+
/**
* Process resource update on a node.
*/
@@ -898,12 +897,12 @@ public abstract class AbstractYarnScheduler
LOG.info("Updated the cluste max priority to maxClusterLevelAppPriority = "
+ maxClusterLevelAppPriority);
}
-
+
/**
* Sanity check increase/decrease request, and return
* SchedulerContainerResourceChangeRequest according to given
* UpdateContainerRequest.
- *
+ *
* <pre>
* - Returns non-null value means validation succeeded
* - Throw exception when any other error happens
@@ -1328,57 +1327,24 @@ public abstract class AbstractYarnScheduler
}
/*
- * Get a Resource object with for the minimum allocation possible. If resource
- * profiles are enabled then the 'minimum' resource profile will be used. If
- * they are not enabled, use the minimums specified in the config files.
+ * Get a Resource object with for the minimum allocation possible.
*
* @return a Resource object with the minimum allocation for the scheduler
*/
public Resource getMinimumAllocation() {
- boolean profilesEnabled = getConfig()
- .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- Resource ret;
- if (!profilesEnabled) {
- ret = ResourceUtils.getResourceTypesMinimumAllocation();
- } else {
- try {
- ret = rmContext.getResourceProfilesManager().getMinimumProfile();
- } catch (YarnException e) {
- LOG.error(
- "Exception while getting minimum profile from profile manager:", e);
- throw new YarnRuntimeException(e);
- }
- }
+ Resource ret = ResourceUtils.getResourceTypesMinimumAllocation();
LOG.info("Minimum allocation = " + ret);
return ret;
}
/**
- * Get a Resource object with for the maximum allocation possible. If resource
- * profiles are enabled then the 'maximum' resource profile will be used. If
- * they are not enabled, use the maximums specified in the config files.
+ * Get a Resource object with for the maximum allocation possible.
*
* @return a Resource object with the maximum allocation for the scheduler
*/
public Resource getMaximumAllocation() {
- boolean profilesEnabled = getConfig()
- .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- Resource ret;
- if (!profilesEnabled) {
- ret = ResourceUtils.getResourceTypesMaximumAllocation();
- } else {
- try {
- ret = rmContext.getResourceProfilesManager().getMaximumProfile();
- } catch (YarnException e) {
- LOG.error(
- "Exception while getting maximum profile from ResourceProfileManager:",
- e);
- throw new YarnRuntimeException(e);
- }
- }
+ Resource ret = ResourceUtils.getResourceTypesMaximumAllocation();
LOG.info("Maximum allocation = " + ret);
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 4a8ff00..b5c20f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -47,14 +47,14 @@ import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.eclipse.jetty.util.log.Log;
public class MockNM {
private int responseId;
private NodeId nodeId;
- private long memory;
- private int vCores;
+ private Resource capatibility;
private ResourceTrackerService resourceTracker;
private int httpPort = 2;
private MasterKey currentContainerTokenMasterKey;
@@ -75,13 +75,25 @@ public class MockNM {
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker) {
- this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion());
+ this(nodeIdStr, memory, vcores, resourceTracker,
+ YarnVersionInfo.getVersion());
}
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker, String version) {
- this.memory = memory;
- this.vCores = vcores;
+ this(nodeIdStr, Resource.newInstance(memory, vcores), resourceTracker,
+ version);
+ }
+
+ public MockNM(String nodeIdStr, Resource capatibility,
+ ResourceTrackerService resourceTracker) {
+ this(nodeIdStr, capatibility, resourceTracker,
+ YarnVersionInfo.getVersion());
+ }
+
+ public MockNM(String nodeIdStr, Resource capatibility,
+ ResourceTrackerService resourceTracker, String version) {
+ this.capatibility = capatibility;
this.resourceTracker = resourceTracker;
this.version = version;
String[] splits = nodeIdStr.split(":");
@@ -146,8 +158,7 @@ public class MockNM {
RegisterNodeManagerRequest.class);
req.setNodeId(nodeId);
req.setHttpPort(httpPort);
- Resource resource = BuilderUtils.newResource(memory, vCores);
- req.setResource(resource);
+ req.setResource(capatibility);
req.setContainerStatuses(containerReports);
req.setNMVersion(version);
req.setRunningApplications(runningApplications);
@@ -158,8 +169,7 @@ public class MockNM {
this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey();
Resource newResource = registrationResponse.getResource();
if (newResource != null) {
- memory = (int) newResource.getMemorySize();
- vCores = newResource.getVirtualCores();
+ capatibility = Resources.clone(newResource);
}
containerStats.clear();
if (containerReports != null) {
@@ -184,7 +194,7 @@ public class MockNM {
long containerId, ContainerState containerState) throws Exception {
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(attemptId, containerId), containerState,
- "Success", 0, BuilderUtils.newResource(memory, vCores));
+ "Success", 0, capatibility);
ArrayList<ContainerStatus> containerStatusList =
new ArrayList<ContainerStatus>(1);
containerStatusList.add(containerStatus);
@@ -264,19 +274,22 @@ public class MockNM {
Resource newResource = heartbeatResponse.getResource();
if (newResource != null) {
- memory = newResource.getMemorySize();
- vCores = newResource.getVirtualCores();
+ capatibility = Resources.clone(newResource);
}
return heartbeatResponse;
}
public long getMemory() {
- return memory;
+ return capatibility.getMemorySize();
}
public int getvCores() {
- return vCores;
+ return capatibility.getVirtualCores();
+ }
+
+ public Resource getCapatibility() {
+ return capatibility;
}
public String getVersion() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 973f487..d128b02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -848,6 +848,15 @@ public class MockRM extends ResourceManager {
return nm;
}
+ public MockNM registerNode(String nodeIdStr, Resource nodeCapatibility)
+ throws Exception {
+ MockNM nm = new MockNM(nodeIdStr, nodeCapatibility,
+ getResourceTrackerService());
+ nm.registerNode();
+ drainEventsImplicitly();
+ return nm;
+ }
+
public void sendNodeStarted(MockNM nm) throws Exception {
RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get(
nm.getNodeId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 7a2e500..421ddbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -676,38 +675,4 @@ public class TestApplicationMasterService {
Assert.fail("Cannot find RMContainer");
}
}
-
- @Test(timeout = 3000000)
- public void testResourceProfiles() throws Exception {
-
- MockRM rm = new MockRM(conf);
- rm.start();
- MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
- RMApp app1 = rm.submitApp(2048);
- nm1.nodeHeartbeat(true);
- RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
- MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
- RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
- Assert.assertEquals(0, resp.getResourceProfiles().size());
- rm.stop();
- conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
- conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
- "profiles/sample-profiles-1.json");
- rm = new MockRM(conf);
- rm.start();
- nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
- app1 = rm.submitApp(2048);
- nm1.nodeHeartbeat(true);
- attempt1 = app1.getCurrentAppAttempt();
- am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
- resp = am1.registerAppAttempt();
- Assert.assertEquals(3, resp.getResourceProfiles().size());
- Assert.assertEquals(Resource.newInstance(1024, 1),
- resp.getResourceProfiles().get("minimum"));
- Assert.assertEquals(Resource.newInstance(2048, 2),
- resp.getResourceProfiles().get("default"));
- Assert.assertEquals(Resource.newInstance(4096, 4),
- resp.getResourceProfiles().get("maximum"));
- rm.stop();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java
new file mode 100644
index 0000000..b75f925
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.resource;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Mock ResourceProfileManager for unit test.
+ */
+public class MockResourceProfileManager extends ResourceProfilesManagerImpl {
+ private Map<String, Resource> profiles;
+ private boolean featureEnabled;
+
+ public MockResourceProfileManager(Map<String, Resource> profiles) {
+ this.profiles = new HashMap<>();
+ this.profiles.putAll(profiles);
+
+ // Set minimum / maximum allocation so test doesn't need to add them
+ // every time.
+ this.profiles.put(ResourceProfilesManagerImpl.MINIMUM_PROFILE,
+ ResourceUtils.getResourceTypesMinimumAllocation());
+ this.profiles.put(ResourceProfilesManagerImpl.MAXIMUM_PROFILE,
+ ResourceUtils.getResourceTypesMaximumAllocation());
+ }
+
+ @Override
+ public void init(Configuration config) throws IOException {
+ this.featureEnabled = config.getBoolean(
+ YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+ }
+
+ @Override
+ public Resource getProfile(String profile) throws YarnException {
+ if (!featureEnabled) {
+ throw new YARNFeatureNotEnabledException("");
+ }
+ return profiles.get(profile);
+ }
+
+ @Override
+ public Map<String, Resource> getResourceProfiles()
+ throws YARNFeatureNotEnabledException {
+ if (!featureEnabled) {
+ throw new YARNFeatureNotEnabledException("");
+ }
+ return profiles;
+ }
+
+ @Override
+ public void reloadProfiles() throws IOException {
+ throw new IOException("Not supported");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
index 291f103..9db129f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
@@ -19,9 +19,15 @@
package org.apache.hadoop.yarn.server.resourcemanager.resource;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.junit.Assert;
import org.junit.Test;
@@ -73,7 +79,7 @@ public class TestResourceProfiles {
Map<String, Resource> expected = new HashMap<>();
expected.put("minimum", Resource.newInstance(1024, 1));
expected.put("default", Resource.newInstance(2048, 2));
- expected.put("maximum", Resource.newInstance(4096, 4));
+ expected.put("maximum", Resource.newInstance(8192, 4));
for (Map.Entry<String, Resource> entry : expected.entrySet()) {
String profile = entry.getKey();
@@ -86,7 +92,7 @@ public class TestResourceProfiles {
}
@Test
- public void testLoadProfilesMissingMandatoryProfile() throws Exception {
+ public void testLoadIllegalProfiles() throws Exception {
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
@@ -115,7 +121,7 @@ public class TestResourceProfiles {
Map<String, Resource> expected = new HashMap<>();
expected.put("minimum", Resource.newInstance(1024, 1));
expected.put("default", Resource.newInstance(2048, 2));
- expected.put("maximum", Resource.newInstance(4096, 4));
+ expected.put("maximum", Resource.newInstance(8192, 4));
expected.put("small", Resource.newInstance(1024, 1));
expected.put("medium", Resource.newInstance(2048, 1));
expected.put("large", Resource.newInstance(4096, 4));
@@ -139,7 +145,7 @@ public class TestResourceProfiles {
Map<String, Resource> expected = new HashMap<>();
expected.put("minimum", Resource.newInstance(1024, 1));
expected.put("default", Resource.newInstance(2048, 2));
- expected.put("maximum", Resource.newInstance(4096, 4));
+ expected.put("maximum", Resource.newInstance(8192, 4));
Assert.assertEquals("Profile 'minimum' resources don't match",
expected.get("minimum"), manager.getMinimumProfile());
@@ -149,4 +155,38 @@ public class TestResourceProfiles {
expected.get("maximum"), manager.getMaximumProfile());
}
+
+ @Test(timeout = 30000)
+ public void testResourceProfilesInAMResponse() throws Exception {
+ Configuration conf = new Configuration();
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024);
+ RMApp app1 = rm.submitApp(2048);
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
+ Assert.assertEquals(0, resp.getResourceProfiles().size());
+ rm.stop();
+ conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+ conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+ "profiles/sample-profiles-1.json");
+ rm = new MockRM(conf);
+ rm.start();
+ nm1 = rm.registerNode("127.0.0.1:1234", 6 * 1024);
+ app1 = rm.submitApp(2048);
+ nm1.nodeHeartbeat(true);
+ attempt1 = app1.getCurrentAppAttempt();
+ am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ resp = am1.registerAppAttempt();
+ Assert.assertEquals(3, resp.getResourceProfiles().size());
+ Assert.assertEquals(Resource.newInstance(1024, 1),
+ resp.getResourceProfiles().get("minimum"));
+ Assert.assertEquals(Resource.newInstance(2048, 2),
+ resp.getResourceProfiles().get("default"));
+ Assert.assertEquals(Resource.newInstance(8192, 4),
+ resp.getResourceProfiles().get("maximum"));
+ rm.stop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
new file mode 100644
index 0000000..b386c18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.MockResourceProfileManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test Capacity Scheduler with multiple resource types.
+ */
+public class TestCapacitySchedulerWithMultiResourceTypes {
+ private static String RESOURCE_1 = "res1";
+ private final int GB = 1024;
+
+ @Test
+ public void testBasicCapacitySchedulerWithProfile() throws Exception {
+
+ // Initialize resource map
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
+ riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
+ riMap.put(RESOURCE_1, ResourceInformation
+ .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
+ Integer.MAX_VALUE));
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f);
+ csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f);
+ csconf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ 100.0f);
+ csconf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f);
+ csconf.setResourceComparator(DominantResourceCalculator.class);
+
+ YarnConfiguration conf = new YarnConfiguration(csconf);
+ // Don't reset resource types since we have already configured resource
+ // types
+ conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+
+ final MockResourceProfileManager mrpm = new MockResourceProfileManager(
+ ImmutableMap.of("res-1", TestUtils
+ .createResource(2 * GB, 2, ImmutableMap.of(RESOURCE_1, 2))));
+
+ MockRM rm = new MockRM(conf) {
+ @Override
+ protected ResourceProfilesManager createResourceProfileManager() {
+ return mrpm;
+ }
+ };
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("default");
+
+ MockNM nm1 = rm.registerNode("h1:1234",
+ TestUtils.createResource(8 * GB, 8, ImmutableMap.of(RESOURCE_1, 8)));
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ Assert.assertEquals(Resource.newInstance(1 * GB, 0),
+ leafQueue.getUsedResources());
+
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ // Now request resource:
+ am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
+ Resource.newInstance(1 * GB, 1)).numContainers(1).resourceName("*")
+ .profileCapability(ProfileCapability
+ .newInstance("res-1",
+ Resource.newInstance(2 * GB, 2))).build()),
+ null);
+
+ // Do node heartbeats 1 time and check container allocated.
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2>
+ Assert.assertEquals(
+ TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+ leafQueue.getUsedResources());
+
+ // Acquire container
+ AllocateResponse amResponse = am1.allocate(null, null);
+ Assert.assertFalse(amResponse.getAllocatedContainers().isEmpty());
+ ContainerTokenIdentifier containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(
+ amResponse.getAllocatedContainers().get(0).getContainerToken());
+ Assert.assertEquals(
+ TestUtils.createResource(2 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+ containerTokenIdentifier.getResource());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 4bc5127..8036a40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -18,16 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
@@ -53,21 +45,28 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import com.google.common.collect.Sets;
-import org.apache.hadoop.yarn.event.Event;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
public class TestUtils {
private static final Log LOG = LogFactory.getLog(TestUtils.class);
@@ -457,4 +456,21 @@ public class TestUtils {
cs.submitResourceCommitRequest(clusterResource,
csAssignment);
}
+
+ /**
+ * An easy way to create resources other than memory and vcores for tests.
+ * @param memory memory
+ * @param vcores vcores
+ * @param nameToValues resource types other than memory and vcores.
+ * @return created resource
+ */
+ public static Resource createResource(long memory, int vcores,
+ Map<String, Integer> nameToValues) {
+ Resource res = Resource.newInstance(memory, vcores);
+ for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) {
+ res.setResourceInformation(entry.getKey(), ResourceInformation
+ .newInstance(entry.getKey(), "", entry.getValue()));
+ }
+ return res;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json
index 65b7360..9107af0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json
@@ -1,14 +1,6 @@
{
- "minimum": {
- "memory-mb" : 1024,
- "vcores" : 1
- },
"default" : {
"memory-mb" : 2048,
"vcores" : 2
- },
- "maximum" : {
- "memory-mb": 4096,
- "vcores" : 4
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cfbad68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json
index c235671..9b03c7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json
@@ -1,16 +1,8 @@
{
- "minimum": {
- "memory-mb" : 1024,
- "vcores" : 1
- },
"default" : {
"memory-mb" : 2048,
"vcores" : 2
},
- "maximum" : {
- "memory-mb": 4096,
- "vcores" : 4
- },
"small" : {
"memory-mb": 1024,
"vcores": 1
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-5946: Create
YarnConfigurationStore interface and InMemoryConfigurationStore class.
Contributed by Jonathan Hung
Posted by su...@apache.org.
YARN-5946: Create YarnConfigurationStore interface and
InMemoryConfigurationStore class. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3579a8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3579a8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3579a8c
Branch: refs/heads/YARN-5881
Commit: e3579a8c3b1dc58a38859b189973be5a2d23f730
Parents: ef59cbe
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 15:58:12 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../conf/InMemoryConfigurationStore.java | 86 +++++++++++
.../capacity/conf/YarnConfigurationStore.java | 154 +++++++++++++++++++
.../conf/TestYarnConfigurationStore.java | 70 +++++++++
3 files changed, 310 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3579a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
new file mode 100644
index 0000000..a208fb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A default implementation of {@link YarnConfigurationStore}. Doesn't offer
+ * persistent configuration storage, just stores the configuration in memory.
+ */
+public class InMemoryConfigurationStore implements YarnConfigurationStore {
+
+ private Configuration schedConf;
+ private LinkedList<LogMutation> pendingMutations;
+ private long pendingId;
+
+ @Override
+ public void initialize(Configuration conf, Configuration schedConf) {
+ this.schedConf = schedConf;
+ this.pendingMutations = new LinkedList<>();
+ this.pendingId = 0;
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation) {
+ logMutation.setId(++pendingId);
+ pendingMutations.add(logMutation);
+ return pendingId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid) {
+ LogMutation mutation = pendingMutations.poll();
+ // If confirmMutation is called out of order, discard mutations until id
+ // is reached.
+ while (mutation != null) {
+ if (mutation.getId() == id) {
+ if (isValid) {
+ Map<String, String> mutations = mutation.getUpdates();
+ for (Map.Entry<String, String> kv : mutations.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ return true;
+ }
+ mutation = pendingMutations.poll();
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ return schedConf;
+ }
+
+ @Override
+ public synchronized List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3579a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
new file mode 100644
index 0000000..22c0ef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YarnConfigurationStore exposes the methods needed for retrieving and
+ * persisting {@link CapacityScheduler} configuration via key-value
+ * using write-ahead logging. When configuration mutation is requested, caller
+ * should first log it with {@code logMutation}, which persists this pending
+ * mutation. This mutation is merged to the persisted configuration only after
+ * {@code confirmMutation} is called.
+ *
+ * On startup/recovery, caller should call {@code retrieve} to get all
+ * confirmed mutations, then get pending mutations which were not confirmed via
+ * {@code getPendingMutations}, and replay/confirm them via
+ * {@code confirmMutation} as in the normal case.
+ */
+public interface YarnConfigurationStore {
+
+ /**
+ * LogMutation encapsulates the fields needed for configuration mutation
+ * audit logging and recovery.
+ */
+ class LogMutation {
+ private Map<String, String> updates;
+ private String user;
+ private long id;
+
+ /**
+ * Create log mutation prior to logging.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ */
+ public LogMutation(Map<String, String> updates, String user) {
+ this(updates, user, 0);
+ }
+
+ /**
+ * Create log mutation for recovery.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ * @param id transaction id of configuration change
+ */
+ LogMutation(Map<String, String> updates, String user, long id) {
+ this.updates = updates;
+ this.user = user;
+ this.id = id;
+ }
+
+ /**
+ * Get key-value configuration updates.
+ * @return map of configuration updates
+ */
+ public Map<String, String> getUpdates() {
+ return updates;
+ }
+
+ /**
+ * Get user who requested configuration change.
+ * @return user who requested configuration change
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * Get transaction id of this configuration change.
+ * @return transaction id
+ */
+ public long getId() {
+ return id;
+ }
+
+ /**
+ * Set transaction id of this configuration change.
+ * @param id transaction id
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+ }
+
+ /**
+ * Initialize the configuration store.
+ * @param conf configuration to initialize store with
+ * @param schedConf Initial key-value configuration to persist
+ */
+ void initialize(Configuration conf, Configuration schedConf);
+
+ /**
+ * Logs the configuration change to backing store. Generates an id associated
+ * with this mutation, sets it in {@code logMutation}, and returns it.
+ * @param logMutation configuration change to be persisted in write ahead log
+ * @return id which configuration store associates with this mutation
+ */
+ long logMutation(LogMutation logMutation);
+
+ /**
+ * Should be called after {@code logMutation}. Gets the pending mutation
+ * associated with {@code id} and marks the mutation as persisted (no longer
+ * pending). If isValid is true, merge the mutation with the persisted
+ * configuration.
+ *
+ * If {@code confirmMutation} is called with ids in a different order than
+ * was returned by {@code logMutation}, the result is implementation
+ * dependent.
+ * @param id id of mutation to be confirmed
+ * @param isValid if true, update persisted configuration with mutation
+ * associated with {@code id}.
+ * @return true on success
+ */
+ boolean confirmMutation(long id, boolean isValid);
+
+ /**
+ * Retrieve the persisted configuration.
+ * @return configuration as key-value
+ */
+ Configuration retrieve();
+
+ /**
+ * Get the list of pending mutations, in the order they were logged.
+ * @return list of mutations
+ */
+ List<LogMutation> getPendingMutations();
+
+ /**
+ * Get a list of confirmed configuration mutations starting from a given id.
+ * @param fromId id from which to start getting mutations, inclusive
+ * @return list of configuration mutations
+ */
+ List<LogMutation> getConfirmedConfHistory(long fromId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3579a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
new file mode 100644
index 0000000..dff4e77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestYarnConfigurationStore {
+
+ private YarnConfigurationStore confStore;
+ private Configuration schedConf;
+
+ private static final String testUser = "testUser";
+
+ @Before
+ public void setUp() {
+ schedConf = new Configuration(false);
+ schedConf.set("key1", "val1");
+ }
+
+ @Test
+ public void testInMemoryConfigurationStore() {
+ confStore = new InMemoryConfigurationStore();
+ confStore.initialize(new Configuration(), schedConf);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ LogMutation mutation1 = new LogMutation(update1, testUser);
+ long id = confStore.logMutation(mutation1);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+ assertEquals(0, confStore.getPendingMutations().size());
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ LogMutation mutation2 = new LogMutation(update2, testUser);
+ id = confStore.logMutation(mutation2);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ assertEquals(0, confStore.getPendingMutations().size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDFS-12635. Unnecessary exception
declaration of the CellBuffers constructor. Contributed by Huafeng Wang
Posted by su...@apache.org.
HDFS-12635. Unnecessary exception declaration of the CellBuffers constructor. Contributed by Huafeng Wang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8d37b63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8d37b63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8d37b63
Branch: refs/heads/YARN-5881
Commit: d8d37b63c71ff4ae0a15d9eb51a9436605ac1a52
Parents: 3d04c00
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Oct 11 15:45:28 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Oct 11 15:45:28 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8d37b63/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 66eec7a..3eb7e67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -198,7 +198,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
private final ByteBuffer[] buffers;
private final byte[][] checksumArrays;
- CellBuffers(int numParityBlocks) throws InterruptedException{
+ CellBuffers(int numParityBlocks) {
if (cellSize % bytesPerChecksum != 0) {
throw new HadoopIllegalArgumentException("Invalid values: "
+ HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY + " (="
@@ -304,12 +304,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
ecPolicy.getCodecName(), coderOptions);
coordinator = new Coordinator(numAllBlocks);
- try {
- cellBuffers = new CellBuffers(numParityBlocks);
- } catch (InterruptedException ie) {
- throw DFSUtilClient.toInterruptedIOException(
- "Failed to create cell buffers", ie);
- }
+ cellBuffers = new CellBuffers(numParityBlocks);
streamers = new ArrayList<>(numAllBlocks);
for (short i = 0; i < numAllBlocks; i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDFS-11575. Supporting HDFS NFS
gateway with Federated HDFS. Contributed by Mukul Kumar Singh.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
index f308763..30ecc0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
@@ -186,7 +186,8 @@ public class TestRpcProgramNfs3 {
public void testGetattr() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
GETATTR3Request req = new GETATTR3Request(handle);
req.serialize(xdr_req);
@@ -209,8 +210,9 @@ public class TestRpcProgramNfs3 {
public void testSetattr() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
SetAttr3 symAttr = new SetAttr3(0, 1, 0, 0, null, null,
EnumSet.of(SetAttrField.UID));
SETATTR3Request req = new SETATTR3Request(handle, symAttr, false, null);
@@ -234,7 +236,8 @@ public class TestRpcProgramNfs3 {
public void testLookup() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
LOOKUP3Request lookupReq = new LOOKUP3Request(handle, "bar");
XDR xdr_req = new XDR();
lookupReq.serialize(xdr_req);
@@ -257,7 +260,8 @@ public class TestRpcProgramNfs3 {
public void testAccess() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
ACCESS3Request req = new ACCESS3Request(handle);
req.serialize(xdr_req);
@@ -281,8 +285,9 @@ public class TestRpcProgramNfs3 {
// Create a symlink first.
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
SYMLINK3Request req = new SYMLINK3Request(handle, "fubar", new SetAttr3(),
"bar");
req.serialize(xdr_req);
@@ -316,7 +321,8 @@ public class TestRpcProgramNfs3 {
public void testRead() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
READ3Request readReq = new READ3Request(handle, 0, 5);
XDR xdr_req = new XDR();
@@ -373,7 +379,8 @@ public class TestRpcProgramNfs3 {
final HdfsFileStatus status = nn.getRpcServer().getFileInfo(fileName);
final long dirId = status.getFileId();
- final FileHandle handle = new FileHandle(dirId);
+ final int namenodeId = Nfs3Utils.getNamenodeId(config);
+ final FileHandle handle = new FileHandle(dirId, namenodeId);
final WRITE3Request writeReq = new WRITE3Request(handle, 0,
buffer.length, WriteStableHow.DATA_SYNC, ByteBuffer.wrap(buffer));
@@ -390,7 +397,8 @@ public class TestRpcProgramNfs3 {
throws Exception {
final HdfsFileStatus status = nn.getRpcServer().getFileInfo(fileName);
final long dirId = status.getFileId();
- final FileHandle handle = new FileHandle(dirId);
+ final int namenodeId = Nfs3Utils.getNamenodeId(config);
+ final FileHandle handle = new FileHandle(dirId, namenodeId);
final READ3Request readReq = new READ3Request(handle, 0, len);
final XDR xdr_req = new XDR();
@@ -422,7 +430,8 @@ public class TestRpcProgramNfs3 {
private void commit(String fileName, int len) throws Exception {
final HdfsFileStatus status = nn.getRpcServer().getFileInfo(fileName);
final long dirId = status.getFileId();
- final FileHandle handle = new FileHandle(dirId);
+ final int namenodeId = Nfs3Utils.getNamenodeId(config);
+ final FileHandle handle = new FileHandle(dirId, namenodeId);
final XDR xdr_req = new XDR();
final COMMIT3Request req = new COMMIT3Request(handle, 0, len);
req.serialize(xdr_req);
@@ -439,7 +448,8 @@ public class TestRpcProgramNfs3 {
public void testWrite() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
byte[] buffer = new byte[10];
for (int i = 0; i < 10; i++) {
@@ -469,8 +479,9 @@ public class TestRpcProgramNfs3 {
public void testCreate() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
CREATE3Request req = new CREATE3Request(handle, "fubar",
Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
req.serialize(xdr_req);
@@ -493,8 +504,9 @@ public class TestRpcProgramNfs3 {
public void testMkdir() throws Exception {//FixME
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
MKDIR3Request req = new MKDIR3Request(handle, "fubar1", new SetAttr3());
req.serialize(xdr_req);
@@ -520,8 +532,9 @@ public class TestRpcProgramNfs3 {
public void testSymlink() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
SYMLINK3Request req = new SYMLINK3Request(handle, "fubar", new SetAttr3(),
"bar");
req.serialize(xdr_req);
@@ -544,8 +557,9 @@ public class TestRpcProgramNfs3 {
public void testRemove() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
REMOVE3Request req = new REMOVE3Request(handle, "bar");
req.serialize(xdr_req);
@@ -567,8 +581,9 @@ public class TestRpcProgramNfs3 {
public void testRmdir() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
RMDIR3Request req = new RMDIR3Request(handle, "foo");
req.serialize(xdr_req);
@@ -590,8 +605,9 @@ public class TestRpcProgramNfs3 {
public void testRename() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
XDR xdr_req = new XDR();
- FileHandle handle = new FileHandle(dirId);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
RENAME3Request req = new RENAME3Request(handle, "bar", handle, "fubar");
req.serialize(xdr_req);
@@ -613,7 +629,8 @@ public class TestRpcProgramNfs3 {
public void testReaddir() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
READDIR3Request req = new READDIR3Request(handle, 0, 0, 100);
req.serialize(xdr_req);
@@ -636,7 +653,8 @@ public class TestRpcProgramNfs3 {
public void testReaddirplus() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo(testdir);
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
READDIRPLUS3Request req = new READDIRPLUS3Request(handle, 0, 0, 3, 2);
req.serialize(xdr_req);
@@ -659,7 +677,8 @@ public class TestRpcProgramNfs3 {
public void testFsstat() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
FSSTAT3Request req = new FSSTAT3Request(handle);
req.serialize(xdr_req);
@@ -682,7 +701,8 @@ public class TestRpcProgramNfs3 {
public void testFsinfo() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
FSINFO3Request req = new FSINFO3Request(handle);
req.serialize(xdr_req);
@@ -705,7 +725,8 @@ public class TestRpcProgramNfs3 {
public void testPathconf() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
PATHCONF3Request req = new PATHCONF3Request(handle);
req.serialize(xdr_req);
@@ -728,7 +749,8 @@ public class TestRpcProgramNfs3 {
public void testCommit() throws Exception {
HdfsFileStatus status = nn.getRpcServer().getFileInfo("/tmp/bar");
long dirId = status.getFileId();
- FileHandle handle = new FileHandle(dirId);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
+ FileHandle handle = new FileHandle(dirId, namenodeId);
XDR xdr_req = new XDR();
COMMIT3Request req = new COMMIT3Request(handle, 0, 5);
req.serialize(xdr_req);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java
new file mode 100644
index 0000000..a5997b4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java
@@ -0,0 +1,330 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.nfs.nfs3;
+
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.viewfs.ConfigUtil;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
+import org.apache.hadoop.hdfs.nfs.mount.RpcProgramMountd;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.nfs.nfs3.FileHandle;
+import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
+import org.apache.hadoop.nfs.nfs3.Nfs3Status;
+import org.apache.hadoop.nfs.nfs3.request.GETATTR3Request;
+import org.apache.hadoop.nfs.nfs3.request.RENAME3Request;
+import org.apache.hadoop.nfs.nfs3.response.GETATTR3Response;
+import org.apache.hadoop.nfs.nfs3.request.WRITE3Request;
+import org.apache.hadoop.nfs.nfs3.response.RENAME3Response;
+import org.apache.hadoop.nfs.nfs3.response.WRITE3Response;
+import org.apache.hadoop.oncrpc.XDR;
+import org.apache.hadoop.oncrpc.security.SecurityHandler;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests for {@link RpcProgramNfs3} with
+ * {@link org.apache.hadoop.fs.viewfs.ViewFileSystem}.
+ */
+public class TestViewfsWithNfs3 {
+ private static DistributedFileSystem hdfs1;
+ private static DistributedFileSystem hdfs2;
+ private static MiniDFSCluster cluster = null;
+ private static NfsConfiguration config = new NfsConfiguration();
+ private static HdfsAdmin dfsAdmin1;
+ private static HdfsAdmin dfsAdmin2;
+ private static FileSystem viewFs;
+
+ private static NameNode nn1;
+ private static NameNode nn2;
+ private static Nfs3 nfs;
+ private static RpcProgramNfs3 nfsd;
+ private static RpcProgramMountd mountd;
+ private static SecurityHandler securityHandler;
+ private static FileSystemTestHelper fsHelper;
+ private static File testRootDir;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ String currentUser = System.getProperty("user.name");
+
+ config.set("fs.permissions.umask-mode", "u=rwx,g=,o=");
+ config.set(DefaultImpersonationProvider.getTestProvider()
+ .getProxySuperuserGroupConfKey(currentUser), "*");
+ config.set(DefaultImpersonationProvider.getTestProvider()
+ .getProxySuperuserIpConfKey(currentUser), "*");
+ fsHelper = new FileSystemTestHelper();
+ // Set up java key store
+ String testRoot = fsHelper.getTestRootDir();
+ testRootDir = new File(testRoot).getAbsoluteFile();
+ final Path jksPath = new Path(testRootDir.toString(), "test.jks");
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
+ ProxyUsers.refreshSuperUserGroupsConfiguration(config);
+
+ cluster =
+ new MiniDFSCluster.Builder(config).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitActive();
+ hdfs1 = cluster.getFileSystem(0);
+ hdfs2 = cluster.getFileSystem(1);
+
+ nn1 = cluster.getNameNode(0);
+ nn2 = cluster.getNameNode(1);
+ nn2.getServiceRpcAddress();
+ dfsAdmin1 = new HdfsAdmin(cluster.getURI(0), config);
+ dfsAdmin2 = new HdfsAdmin(cluster.getURI(1), config);
+
+ // Use ephemeral ports in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+ config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.VIEWFS_URI.toString());
+ // Start NFS with allowed.hosts set to "* rw"
+ config.set("dfs.nfs.exports.allowed.hosts", "* rw");
+
+ Path base1 = new Path("/user1");
+ Path base2 = new Path("/user2");
+ hdfs1.delete(base1, true);
+ hdfs2.delete(base2, true);
+ hdfs1.mkdirs(base1);
+ hdfs2.mkdirs(base2);
+ ConfigUtil.addLink(config, "/hdfs1", hdfs1.makeQualified(base1).toUri());
+ ConfigUtil.addLink(config, "/hdfs2", hdfs2.makeQualified(base2).toUri());
+
+
+ viewFs = FileSystem.get(config);
+ config.setStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY,
+ "/hdfs1", "/hdfs2");
+
+ nfs = new Nfs3(config);
+ nfs.startServiceInternal(false);
+ nfsd = (RpcProgramNfs3) nfs.getRpcProgram();
+ mountd = (RpcProgramMountd) nfs.getMountd().getRpcProgram();
+
+ // Mock SecurityHandler which returns system user.name
+ securityHandler = Mockito.mock(SecurityHandler.class);
+ Mockito.when(securityHandler.getUser()).thenReturn(currentUser);
+ viewFs.delete(new Path("/hdfs2/dir2"), true);
+ viewFs.mkdirs(new Path("/hdfs2/dir2"));
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs1/file1"), 0, (short) 1, 0);
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs1/file2"), 0, (short) 1, 0);
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs1/write1"), 0, (short) 1, 0);
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs2/write2"), 0, (short) 1, 0);
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameMultiNN"),
+ 0, (short) 1, 0);
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameSingleNN"),
+ 0, (short) 1, 0);
+ }
+
+ @AfterClass
+ public static void shutdown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testNumExports() throws Exception {
+ Assert.assertEquals(mountd.getExports().size(),
+ viewFs.getChildFileSystems().length);
+ }
+
+ @Test
+ public void testPaths() throws Exception {
+ Assert.assertEquals(hdfs1.resolvePath(new Path("/user1/file1")),
+ viewFs.resolvePath(new Path("/hdfs1/file1")));
+ Assert.assertEquals(hdfs1.resolvePath(new Path("/user1/file2")),
+ viewFs.resolvePath(new Path("/hdfs1/file2")));
+ Assert.assertEquals(hdfs2.resolvePath(new Path("/user2/dir2")),
+ viewFs.resolvePath(new Path("/hdfs2/dir2")));
+ }
+
+ @Test
+ public void testFileStatus() throws Exception {
+ HdfsFileStatus status = nn1.getRpcServer().getFileInfo("/user1/file1");
+ FileStatus st = viewFs.getFileStatus(new Path("/hdfs1/file1"));
+ Assert.assertEquals(st.isDirectory(), status.isDirectory());
+
+ HdfsFileStatus status2 = nn2.getRpcServer().getFileInfo("/user2/dir2");
+ FileStatus st2 = viewFs.getFileStatus(new Path("/hdfs2/dir2"));
+ Assert.assertEquals(st2.isDirectory(), status2.isDirectory());
+ }
+
+ // Test for getattr
+ private void testNfsGetAttrResponse(long fileId, int namenodeId,
+ int expectedStatus) {
+ FileHandle handle = new FileHandle(fileId, namenodeId);
+ XDR xdrReq = new XDR();
+ GETATTR3Request req = new GETATTR3Request(handle);
+ req.serialize(xdrReq);
+ GETATTR3Response response = nfsd.getattr(xdrReq.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ Assert.assertEquals("Incorrect return code",
+ expectedStatus, response.getStatus());
+ }
+
+ @Test (timeout = 60000)
+ public void testNfsAccessNN1() throws Exception {
+ HdfsFileStatus status = nn1.getRpcServer().getFileInfo("/user1/file1");
+ int namenodeId = Nfs3Utils.getNamenodeId(config, hdfs1.getUri());
+ testNfsGetAttrResponse(status.getFileId(), namenodeId, Nfs3Status.NFS3_OK);
+ }
+
+ @Test (timeout = 60000)
+ public void testNfsAccessNN2() throws Exception {
+ HdfsFileStatus status = nn2.getRpcServer().getFileInfo("/user2/dir2");
+ int namenodeId = Nfs3Utils.getNamenodeId(config, hdfs2.getUri());
+ testNfsGetAttrResponse(status.getFileId(), namenodeId, Nfs3Status.NFS3_OK);
+ }
+
+ @Test (timeout = 60000)
+ public void testWrongNfsAccess() throws Exception {
+ DFSTestUtil.createFile(viewFs, new Path("/hdfs1/file3"), 0, (short) 1, 0);
+ HdfsFileStatus status = nn1.getRpcServer().getFileInfo("/user1/file3");
+ int namenodeId = Nfs3Utils.getNamenodeId(config, hdfs2.getUri());
+ testNfsGetAttrResponse(status.getFileId(), namenodeId,
+ Nfs3Status.NFS3ERR_IO);
+ }
+
+ // Test for write
+ private void testNfsWriteResponse(long dirId, int namenodeId)
+ throws Exception {
+ FileHandle handle = new FileHandle(dirId, namenodeId);
+
+ byte[] buffer = new byte[10];
+ for (int i = 0; i < 10; i++) {
+ buffer[i] = (byte) i;
+ }
+
+ WRITE3Request writeReq = new WRITE3Request(handle, 0, 10,
+ Nfs3Constant.WriteStableHow.DATA_SYNC, ByteBuffer.wrap(buffer));
+ XDR xdrReq = new XDR();
+ writeReq.serialize(xdrReq);
+
+ // Attempt by a priviledged user should pass.
+ WRITE3Response response = nfsd.write(xdrReq.asReadOnlyWrap(),
+ null, 1, securityHandler,
+ new InetSocketAddress("localhost", 1234));
+ Assert.assertEquals("Incorrect response:", null, response);
+ }
+
+ @Test (timeout = 60000)
+ public void testNfsWriteNN1() throws Exception {
+ HdfsFileStatus status = nn1.getRpcServer().getFileInfo("/user1/write1");
+ int namenodeId = Nfs3Utils.getNamenodeId(config, hdfs1.getUri());
+ testNfsWriteResponse(status.getFileId(), namenodeId);
+ }
+
+ @Test (timeout = 60000)
+ public void testNfsWriteNN2() throws Exception {
+ HdfsFileStatus status = nn2.getRpcServer().getFileInfo("/user2/write2");
+ int namenodeId = Nfs3Utils.getNamenodeId(config, hdfs2.getUri());
+ testNfsWriteResponse(status.getFileId(), namenodeId);
+ }
+
+ // Test for rename
+ private void testNfsRename(FileHandle fromDirHandle, String fromFileName,
+ FileHandle toDirHandle, String toFileName,
+ int expectedStatus) throws Exception {
+ XDR xdrReq = new XDR();
+ RENAME3Request req = new RENAME3Request(fromDirHandle, fromFileName,
+ toDirHandle, toFileName);
+ req.serialize(xdrReq);
+
+ // Attempt by a privileged user should pass.
+ RENAME3Response response = nfsd.rename(xdrReq.asReadOnlyWrap(),
+ securityHandler, new InetSocketAddress("localhost", 1234));
+ assertEquals(expectedStatus, response.getStatus());
+ }
+
+ @Test (timeout = 60000)
+ public void testNfsRenameMultiNN() throws Exception {
+ HdfsFileStatus fromFileStatus = nn1.getRpcServer().getFileInfo("/user1");
+ int fromNNId = Nfs3Utils.getNamenodeId(config, hdfs1.getUri());
+ FileHandle fromHandle =
+ new FileHandle(fromFileStatus.getFileId(), fromNNId);
+
+ HdfsFileStatus toFileStatus = nn2.getRpcServer().getFileInfo("/user2");
+ int toNNId = Nfs3Utils.getNamenodeId(config, hdfs2.getUri());
+ FileHandle toHandle = new FileHandle(toFileStatus.getFileId(), toNNId);
+
+ HdfsFileStatus statusBeforeRename =
+ nn1.getRpcServer().getFileInfo("/user1/renameMultiNN");
+ Assert.assertEquals(statusBeforeRename.isDirectory(), false);
+
+ testNfsRename(fromHandle, "renameMultiNN",
+ toHandle, "renameMultiNNFail", Nfs3Status.NFS3ERR_INVAL);
+
+ HdfsFileStatus statusAfterRename =
+ nn2.getRpcServer().getFileInfo("/user2/renameMultiNNFail");
+ Assert.assertEquals(statusAfterRename, null);
+
+ statusAfterRename = nn1.getRpcServer().getFileInfo("/user1/renameMultiNN");
+ Assert.assertEquals(statusAfterRename.isDirectory(), false);
+ }
+
+ @Test (timeout = 60000)
+ public void testNfsRenameSingleNN() throws Exception {
+ HdfsFileStatus fromFileStatus = nn1.getRpcServer().getFileInfo("/user1");
+ int fromNNId = Nfs3Utils.getNamenodeId(config, hdfs1.getUri());
+ FileHandle fromHandle =
+ new FileHandle(fromFileStatus.getFileId(), fromNNId);
+
+ HdfsFileStatus statusBeforeRename =
+ nn1.getRpcServer().getFileInfo("/user1/renameSingleNN");
+ Assert.assertEquals(statusBeforeRename.isDirectory(), false);
+
+ testNfsRename(fromHandle, "renameSingleNN",
+ fromHandle, "renameSingleNNSucess", Nfs3Status.NFS3_OK);
+
+ HdfsFileStatus statusAfterRename =
+ nn1.getRpcServer().getFileInfo("/user1/renameSingleNNSucess");
+ Assert.assertEquals(statusAfterRename.isDirectory(), false);
+
+ statusAfterRename =
+ nn1.getRpcServer().getFileInfo("/user1/renameSingleNN");
+ Assert.assertEquals(statusAfterRename, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6602b5f/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
index 9c327c4..f7a92fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
@@ -481,6 +481,7 @@ public class TestWrites {
cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
cluster.waitActive();
client = new DFSClient(DFSUtilClient.getNNAddress(config), config);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
// Use emphral port in case tests are running in parallel
config.setInt("nfs3.mountd.port", 0);
@@ -492,7 +493,7 @@ public class TestWrites {
nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
HdfsFileStatus status = client.getFileInfo("/");
- FileHandle rootHandle = new FileHandle(status.getFileId());
+ FileHandle rootHandle = new FileHandle(status.getFileId(), namenodeId);
// Create file1
CREATE3Request createReq = new CREATE3Request(rootHandle, "file1",
Nfs3Constant.CREATE_UNCHECKED, new SetAttr3(), 0);
@@ -598,8 +599,9 @@ public class TestWrites {
DFSClient dfsClient = new DFSClient(DFSUtilClient.getNNAddress(config),
config);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
HdfsFileStatus status = dfsClient.getFileInfo("/");
- FileHandle rootHandle = new FileHandle(status.getFileId());
+ FileHandle rootHandle = new FileHandle(status.getFileId(), namenodeId);
CREATE3Request createReq = new CREATE3Request(rootHandle,
"out-of-order-write" + System.currentTimeMillis(),
@@ -674,8 +676,9 @@ public class TestWrites {
DFSClient dfsClient = new DFSClient(DFSUtilClient.getNNAddress(config),
config);
+ int namenodeId = Nfs3Utils.getNamenodeId(config);
HdfsFileStatus status = dfsClient.getFileInfo("/");
- FileHandle rootHandle = new FileHandle(status.getFileId());
+ FileHandle rootHandle = new FileHandle(status.getFileId(), namenodeId);
CREATE3Request createReq = new CREATE3Request(rootHandle,
"overlapping-writes" + System.currentTimeMillis(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDFS-12541. Extend
TestSafeModeWithStripedFile with a random EC policy. Contributed by Takanobu
Asanuma.
Posted by su...@apache.org.
HDFS-12541. Extend TestSafeModeWithStripedFile with a random EC policy. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79382082
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79382082
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79382082
Branch: refs/heads/YARN-5881
Commit: 793820823325e390bb671f4cc1b3bc6920bba5de
Parents: afb42ae
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Oct 9 13:03:32 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Oct 9 13:03:32 2017 -0700
----------------------------------------------------------------------
.../hdfs/TestSafeModeWithStripedFile.java | 29 ++++++++----
...feModeWithStripedFileWithRandomECPolicy.java | 49 ++++++++++++++++++++
2 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79382082/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
index 3d3ec9c..a43cc52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
@@ -42,13 +42,12 @@ import static org.junit.Assert.assertTrue;
public class TestSafeModeWithStripedFile {
- private final ErasureCodingPolicy ecPolicy =
- StripedFileTestUtil.getDefaultECPolicy();
- private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
- private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
- private final int numDNs = dataBlocks + parityBlocks;
- private final int cellSize = ecPolicy.getCellSize();
- private final int blockSize = cellSize * 2;
+ private ErasureCodingPolicy ecPolicy;
+ private short dataBlocks;
+ private short parityBlocks;
+ private int numDNs;
+ private int cellSize;
+ private int blockSize;
private MiniDFSCluster cluster;
private Configuration conf;
@@ -56,17 +55,27 @@ public class TestSafeModeWithStripedFile {
@Rule
public Timeout globalTimeout = new Timeout(300000);
+ public ErasureCodingPolicy getEcPolicy() {
+ return StripedFileTestUtil.getDefaultECPolicy();
+ }
+
@Before
public void setup() throws IOException {
+ ecPolicy = getEcPolicy();
+ dataBlocks = (short) ecPolicy.getNumDataUnits();
+ parityBlocks = (short) ecPolicy.getNumParityUnits();
+ numDNs = dataBlocks + parityBlocks;
+ cellSize = ecPolicy.getCellSize();
+ blockSize = cellSize * 2;
+
conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().enableErasureCodingPolicy(getEcPolicy().getName());
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
- StripedFileTestUtil.getDefaultECPolicy().getName());
+ getEcPolicy().getName());
cluster.waitActive();
- cluster.getFileSystem().enableErasureCodingPolicy(
- StripedFileTestUtil.getDefaultECPolicy().getName());
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79382082/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java
new file mode 100644
index 0000000..00c29dc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test extends TestSafeModeWithStripedFile to use a random
+ * (non-default) EC policy.
+ */
+public class TestSafeModeWithStripedFileWithRandomECPolicy extends
+ TestSafeModeWithStripedFile {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestSafeModeWithStripedFileWithRandomECPolicy.class);
+
+ private ErasureCodingPolicy ecPolicy;
+
+ public TestSafeModeWithStripedFileWithRandomECPolicy() {
+ // If you want to debug this test with a specific ec policy, please use
+ // SystemErasureCodingPolicies class.
+ // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
+ ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
+ LOG.info("run {} with {}.",
+ TestSafeModeWithStripedFileWithRandomECPolicy.class
+ .getSuperclass().getSimpleName(), ecPolicy.getName());
+ }
+
+ @Override
+ public ErasureCodingPolicy getEcPolicy() {
+ return ecPolicy;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 0544387..4985a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
import java.util.Collections;
@@ -121,6 +123,15 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
// For queue with same used ratio / priority, queue with higher configured
// capacity goes first
if (0 == rc) {
+ Resource minEffRes1 = q1.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ Resource minEffRes2 = q2.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ if (!minEffRes1.equals(Resources.none())
+ && !minEffRes2.equals(Resources.none())) {
+ return minEffRes2.compareTo(minEffRes1);
+ }
+
float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p);
float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p);
return Float.compare(abs2, abs1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 22705cc..86b2fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -62,6 +62,8 @@ public class CapacitySchedulerQueueInfo {
protected long pendingContainers;
protected QueueCapacitiesInfo capacities;
protected ResourcesInfo resources;
+ protected ResourceInfo minEffectiveCapacity;
+ protected ResourceInfo maxEffectiveCapacity;
CapacitySchedulerQueueInfo() {
};
@@ -105,6 +107,11 @@ public class CapacitySchedulerQueueInfo {
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
+
+ minEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMinResource());
+ maxEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMaxResource());
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -200,4 +207,12 @@ public class CapacitySchedulerQueueInfo {
public ResourcesInfo getResources() {
return resources;
}
+
+ public ResourceInfo getMinEffectiveCapacity(){
+ return minEffectiveCapacity;
+ }
+
+ public ResourceInfo getMaxEffectiveCapacity(){
+ return maxEffectiveCapacity;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index b5c20f4..909d023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -142,6 +143,13 @@ public class MockNM {
return this.registeringCollectors;
}
+ public void unRegisterNode() throws Exception {
+ UnRegisterNodeManagerRequest request = Records
+ .newRecord(UnRegisterNodeManagerRequest.class);
+ request.setNodeId(nodeId);
+ resourceTracker.unRegisterNodeManager(request);
+ }
+
public RegisterNodeManagerResponse registerNode() throws Exception {
return registerNode(null, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index d128b02..d982d2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -822,6 +822,12 @@ public class MockRM extends ResourceManager {
return rmApp;
}
+ public MockNM unRegisterNode(MockNM nm) throws Exception {
+ nm.unRegisterNode();
+ drainEventsImplicitly();
+ return nm;
+ }
+
public MockNM registerNode(String nodeIdStr, int memory) throws Exception {
MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService());
nm.registerNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 4fc0ea4..591d5f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
@@ -641,9 +642,11 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
QueueCapacities qc = new QueueCapacities(0 == myLevel);
ResourceUsage ru = new ResourceUsage();
+ QueueResourceQuotas qr = new QueueResourceQuotas();
when(queue.getQueueCapacities()).thenReturn(qc);
when(queue.getQueueResourceUsage()).thenReturn(ru);
+ when(queue.getQueueResourceQuotas()).thenReturn(qr);
LOG.debug("Setup queue, name=" + queue.getQueueName() + " path="
+ queue.getQueuePath());
@@ -676,7 +679,17 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
qc.setAbsoluteMaximumCapacity(partitionName, absMax);
qc.setAbsoluteUsedCapacity(partitionName, absUsed);
qc.setUsedCapacity(partitionName, used);
+ qr.setEffectiveMaxResource(parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(parseResourceFromString(values[0].trim()));
+ qr.setEffectiveMaxResource(partitionName,
+ parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(partitionName,
+ parseResourceFromString(values[0].trim()));
when(queue.getUsedCapacity()).thenReturn(used);
+ when(queue.getEffectiveCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[0].trim()));
+ when(queue.getEffectiveMaxCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[1].trim()));
ru.setPending(partitionName, pending);
// Setup reserved resource if it contained by input config
Resource reserved = Resources.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index a14a2b1..b881323 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.Clock;
@@ -435,8 +435,8 @@ public class TestProportionalCapacityPreemptionPolicy {
policy.editSchedule();
// queueF(appD) wants resources, Verify that resources come from queueE(appC)
// because it's a sibling and queueB(appA) because queueA is over capacity.
- verify(mDisp, times(28)).handle(argThat(new IsPreemptionRequestFor(appA)));
- verify(mDisp, times(22)).handle(argThat(new IsPreemptionRequestFor(appC)));
+ verify(mDisp, times(27)).handle(argThat(new IsPreemptionRequestFor(appA)));
+ verify(mDisp, times(23)).handle(argThat(new IsPreemptionRequestFor(appC)));
// Need to call setup() again to reset mDisp
setup();
@@ -1170,6 +1170,17 @@ public class TestProportionalCapacityPreemptionPolicy {
when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
boolean preemptionDisabled = mockPreemptionStatus("root");
when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
+ QueueResourceQuotas rootQr = new QueueResourceQuotas();
+ rootQr.setEffectiveMaxResource(Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(abs[0]);
+ rootQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[0]);
+ when(root.getQueueResourceQuotas()).thenReturn(rootQr);
+ when(root.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[0]);
+ when(root.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[0], 0));
for (int i = 1; i < queues.length; ++i) {
final CSQueue q;
@@ -1200,6 +1211,18 @@ public class TestProportionalCapacityPreemptionPolicy {
qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
when(q.getQueueCapacities()).thenReturn(qc);
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ qr.setEffectiveMaxResource(Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(abs[i]);
+ qr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]);
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
+ when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[i]);
+ when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[i], 0));
+
String parentPathName = p.getQueuePath();
parentPathName = (parentPathName == null) ? "root" : parentPathName;
String queuePathName = (parentPathName + "." + queueName).replace("/",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
index 7784549..a1d89d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
@@ -67,9 +67,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY,
"priority_first");
- String labelsConfig = "=100:200,true;";
+ String labelsConfig = "=100:50,true;";
String nodesConfig = // n1 has no label
- "n1= res=100:200";
+ "n1= res=100:50";
String queuesConfig =
// guaranteed,max,used,pending,reserved
"root(=[100:50 100:50 80:40 120:60 0]);" + // root
@@ -105,7 +105,7 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
- verify(mDisp, times(7)).handle(argThat(
+ verify(mDisp, times(3)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
new file mode 100644
index 0000000..5a66281
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -0,0 +1,516 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAbsoluteResourceConfiguration {
+
+ private static final int GB = 1024;
+
+ private static final String QUEUEA = "queueA";
+ private static final String QUEUEB = "queueB";
+ private static final String QUEUEC = "queueC";
+ private static final String QUEUEA1 = "queueA1";
+ private static final String QUEUEA2 = "queueA2";
+ private static final String QUEUEB1 = "queueB1";
+
+ private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEA;
+ private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEB;
+ private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEC;
+ private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1;
+ private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2;
+ private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1;
+
+ private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB,
+ 10);
+ private static final Resource QUEUE_A_MAXRES = Resource.newInstance(200 * GB,
+ 30);
+ private static final Resource QUEUE_A1_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_A2_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_B_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_B1_MINRES = Resource.newInstance(40 * GB,
+ 10);
+ private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB,
+ 30);
+ private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB,
+ 20);
+ private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6);
+ private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000,
+ 20);
+
+ private static Set<String> resourceTypes = new HashSet<>(
+ Arrays.asList("memory", "vcores"));
+
+ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+ csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ csConf.setCapacity(QUEUEA1_FULL, 50f);
+ csConf.setCapacity(QUEUEA2_FULL, 50f);
+ csConf.setCapacity(QUEUEB1_FULL, 100f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ @Test
+ public void testSimpleMinMaxResourceConfigurartionPerQueue() {
+
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true);
+ setupMinMaxResourceConfiguration(csConf);
+
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ }
+
+ @Test
+ public void testEffectiveMinMaxResourceConfigurartionPerQueue()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+
+ @Test
+ public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases 1) Configure percentage based capacity and absolute
+ * resource together. 2) As per above tree structure, ensure all values
+ * could be retrieved. 3) Validate whether min resource cannot be more than
+ * max resources. 4) Validate whether max resource of queue cannot be more
+ * than its parent max resource.
+ */
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // 1. Create a new config with capcity and min/max together. Ensure an
+ // exception is thrown.
+ CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
+ true);
+ setupMinMaxResourceConfiguration(csConf1);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Queue 'queueA' should use either"
+ + " percentage based capacity configuration or absolute resource.",
+ e.getMessage());
+ }
+ rm.stop();
+
+ // 2. Create a new config with min/max alone with a complex queue config.
+ // Check all values could be fetched correctly.
+ CapacitySchedulerConfiguration csConf2 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf2);
+
+ rm = new MockRM(csConf2);
+ rm.start();
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+ cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+ Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+ QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+ QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+ Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+ QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+ QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+ Assert.assertNotNull(qB1);
+ Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // 3. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf3 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf3);
+
+ csConf3.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B_MAXRES);
+ csConf3.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ try {
+ cs.reinitialize(csConf3, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Min resource configuration "
+ + "<memory:153600, vCores:30> is greater than its "
+ + "max value:<memory:40960, vCores:10> in queue:queueB1",
+ e.getMessage());
+ }
+
+ // 4. Create a new config and make sure one queue's max resource is more
+ // than its preant's max resource configured.
+ CapacitySchedulerConfiguration csConf4 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf4);
+
+ csConf4.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_A_MAXRES);
+
+ try {
+ cs.reinitialize(csConf4, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert
+ .assertEquals(
+ "Failed to re-init queues : Max resource configuration "
+ + "<memory:204800, vCores:30> is greater than parents max value:"
+ + "<memory:153600, vCores:30> in queue:queueB1",
+ e.getMessage());
+ }
+ rm.stop();
+ }
+
+ @Test
+ public void testComplexValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases: 1) Parent and its child queues must use either
+ * percentage based or absolute resource configuration. 2) Parent's min
+ * resource must be more than sum of child's min resource.
+ */
+
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // 1. Explicitly set percentage based config for parent queues. This will
+ // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
+ // resource.
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+
+ // Also unset resource based config.
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ try {
+ cs.reinitialize(csConf, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Parent queue 'queueA' "
+ + "and child queue 'queueA1' should use either percentage based"
+ + " capacity configuration or absolute resource together.",
+ e.getMessage());
+ }
+
+ // 2. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf1 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf1);
+
+ // Configure QueueA with lesser resource than its children.
+ csConf1.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A1_MINRES);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals("Failed to re-init queues : Parent Queues capacity: "
+ + "<memory:51200, vCores:5> is less than to its children:"
+ + "<memory:102400, vCores:10> for queue:queueA", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEffectiveResourceAfterReducingClusterResource()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+ rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // unregister one NM.
+ rm.unRegisterNode(nm1);
+
+ // After loosing one NM, effective min res of queueA will become just
+ // above half. Hence A's min will be 60Gi and 6 cores and max will be
+ // 128GB and 20 cores.
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 8aca235..24ae244 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -86,6 +87,7 @@ public class TestApplicationLimits {
final static int GB = 1024;
LeafQueue queue;
+ CSQueue root;
private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -100,7 +102,7 @@ public class TestApplicationLimits {
setupQueueConfiguration(csConf);
rmContext = TestUtils.getMockRMContext();
-
+ Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32);
CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
when(csContext.getConfiguration()).thenReturn(csConf);
@@ -110,10 +112,11 @@ public class TestApplicationLimits {
when(csContext.getMaximumResourceCapability()).
thenReturn(Resources.createResource(16*GB, 32));
when(csContext.getClusterResource()).
- thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
+ thenReturn(clusterResource);
when(csContext.getResourceCalculator()).
thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
RMContainerTokenSecretManager containerTokenSecretManager =
new RMContainerTokenSecretManager(conf);
@@ -122,13 +125,17 @@ public class TestApplicationLimits {
containerTokenSecretManager);
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
- CSQueue root = CapacitySchedulerQueueManager
+ root = CapacitySchedulerQueueManager
.parseQueue(csContext, csConf, null, "root",
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
-
queue = spy(new LeafQueue(csContext, A, root, null));
+ QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A))
+ .getQueueResourceQuotas();
+ doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas();
// Stub out ACL checks
doReturn(true).
@@ -189,6 +196,8 @@ public class TestApplicationLimits {
// when there is only 1 user, and drops to 2G (the userlimit) when there
// is a second user
Resource clusterResource = Resource.newInstance(80 * GB, 40);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue.updateClusterResource(clusterResource, new ResourceLimits(
clusterResource));
@@ -287,6 +296,8 @@ public class TestApplicationLimits {
CSQueue root =
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
"root", queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
LeafQueue queue = (LeafQueue)queues.get(A);
@@ -357,6 +368,8 @@ public class TestApplicationLimits {
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
clusterResource = Resources.createResource(100 * 16 * GB);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
@@ -378,6 +391,8 @@ public class TestApplicationLimits {
root = CapacitySchedulerQueueManager.parseQueue(
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
@@ -393,7 +408,7 @@ public class TestApplicationLimits {
final String user_0 = "user_0";
final String user_1 = "user_1";
final String user_2 = "user_2";
-
+
assertEquals(Resource.newInstance(16 * GB, 1),
queue.calculateAndGetAMResourceLimit());
assertEquals(Resource.newInstance(8 * GB, 1),
@@ -578,6 +593,7 @@ public class TestApplicationLimits {
thenReturn(Resources.createResource(16*GB));
when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
// Say cluster has 100 nodes of 16G each
Resource clusterResource = Resources.createResource(100 * 16 * GB);
@@ -586,6 +602,8 @@ public class TestApplicationLimits {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -693,6 +711,8 @@ public class TestApplicationLimits {
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.createResource(90*16*GB);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Any change is cluster resource needs to enforce user-limit recomputation.
// In existing code, LeafQueue#updateClusterResource handled this. However
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index 0aac2ef..d73f1c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -600,6 +601,7 @@ public class TestApplicationLimitsByPartition {
RMContext spyRMContext = spy(rmContext);
when(spyRMContext.getNodeLabelManager()).thenReturn(mgr);
when(csContext.getRMContext()).thenReturn(spyRMContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
mgr.activateNode(NodeId.newInstance("h0", 0),
Resource.newInstance(160 * GB, 16)); // default Label
@@ -615,6 +617,8 @@ public class TestApplicationLimitsByPartition {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 1dea4ee..70f2b02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4305,7 +4305,7 @@ public class TestCapacityScheduler {
null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
- assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
+ assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
cs.getQueue("p2").getUsedResources().getMemorySize());
//Free a container from X1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index e34665d..b6b0361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -242,6 +242,8 @@ public class TestChildQueueOrder {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index a32352b..96f8970 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -191,6 +191,8 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration.ROOT,
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32),
+ new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32)));
ResourceUsage queueResUsage = root.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -307,13 +309,11 @@ public class TestLeafQueue {
// Verify the value for getAMResourceLimit for queues with < .1 maxcap
Resource clusterResource = Resource.newInstance(50 * GB, 50);
- a.updateClusterResource(clusterResource,
+ root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
assertEquals(Resource.newInstance(1 * GB, 1),
a.calculateAndGetAMResourceLimit());
- b.updateClusterResource(clusterResource,
- new ResourceLimits(clusterResource));
assertEquals(Resource.newInstance(5 * GB, 1),
b.calculateAndGetAMResourceLimit());
}
@@ -358,6 +358,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -556,6 +558,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -630,6 +634,8 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -699,6 +705,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (80 * GB), numNodes * 100);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Set user-limit. Need a small queue within a large cluster.
b.setUserLimit(50);
@@ -779,6 +787,8 @@ public class TestLeafQueue {
Resources.createResource(numNodes * (8 * GB), numNodes * 100);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getClusterResource()).thenReturn(clusterResource);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests so that one application is memory dominant
// and other application is vcores dominant
@@ -891,6 +901,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -915,6 +927,8 @@ public class TestLeafQueue {
// Set user-limit
a.setUserLimit(50);
a.setUserLimitFactor(2);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// There're two active users
assertEquals(2, a.getAbstractUsersManager().getNumActiveUsers());
@@ -940,7 +954,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
// Allocate one container to app_0, before allocating this container,
- // user-limit = ceil((4 + 1) / 2) = 3G. app_0's used resource (3G) <=
+ // user-limit = floor((5 + 1) / 2) = 3G. app_0's used resource (3G) <=
// user-limit.
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_1,
@@ -1068,15 +1082,9 @@ public class TestLeafQueue {
a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(9*GB, a.getUsedResources().getMemorySize());
- assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-
- assertEquals(4*GB,
- app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize());
-
- assertEquals(1*GB,
- app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize());
+ assertEquals(12*GB, a.getUsedResources().getMemorySize());
+ assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize());
+ assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -1100,6 +1108,8 @@ public class TestLeafQueue {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
CapacitySchedulerQueueManager mockCapacitySchedulerQueueManager
= mock(CapacitySchedulerQueueManager.class);
@@ -1122,6 +1132,8 @@ public class TestLeafQueue {
qb.setUserLimit(100);
qb.setUserLimitFactor(1);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
final ApplicationAttemptId appAttemptId_0 =
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
@@ -1256,106 +1268,6 @@ public class TestLeafQueue {
}
@Test
- public void testUserHeadroomMultiApp() throws Exception {
- // Mock the queue
- LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
- //unset maxCapacity
- a.setMaxCapacity(1.0f);
-
- // Users
- final String user_0 = "user_0";
- final String user_1 = "user_1";
-
- // Submit applications
- final ApplicationAttemptId appAttemptId_0 =
- TestUtils.getMockApplicationAttemptId(0, 0);
- FiCaSchedulerApp app_0 =
- new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_0, user_0);
-
- final ApplicationAttemptId appAttemptId_1 =
- TestUtils.getMockApplicationAttemptId(1, 0);
- FiCaSchedulerApp app_1 =
- new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_1, user_0); // same user
-
- final ApplicationAttemptId appAttemptId_2 =
- TestUtils.getMockApplicationAttemptId(2, 0);
- FiCaSchedulerApp app_2 =
- new FiCaSchedulerApp(appAttemptId_2, user_1, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_2, user_1);
-
- // Setup some nodes
- String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK,
- 0, 16*GB);
- String host_1 = "127.0.0.2";
- FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK,
- 0, 16*GB);
-
- Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(
- app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(),
- app_1, app_2.getApplicationAttemptId(), app_2);
- Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(),
- node_0, node_1.getNodeID(), node_1);
-
- final int numNodes = 2;
- Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1);
- when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
- Priority priority = TestUtils.createMockPriority(1);
-
- app_0.updateResourceRequests(Collections.singletonList(
- TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
- priority, recordFactory)));
-
- applyCSAssignment(clusterResource,
- a.assignContainers(clusterResource, node_0,
- new ResourceLimits(clusterResource),
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(1*GB, a.getUsedResources().getMemorySize());
- assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
- //Now, headroom is the same for all apps for a given user + queue combo
- //and a change to any app's headroom is reflected for all the user's apps
- //once those apps are active/have themselves calculated headroom for
- //allocation at least one time
- assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize());//not yet active
- assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
- app_1.updateResourceRequests(Collections.singletonList(
- TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
-
- applyCSAssignment(clusterResource,
- a.assignContainers(clusterResource, node_0,
- new ResourceLimits(clusterResource),
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(2*GB, a.getUsedResources().getMemorySize());
- assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(1*GB, app_1.getHeadroom().getMemorySize());//now active
- assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
- //Complete container and verify that headroom is updated, for both apps
- //for the user
- RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
- a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- ContainerStatus.newInstance(rmContainer.getContainerId(),
- ContainerState.COMPLETE, "",
- ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
- RMContainerEventType.KILL, null, true);
-
- assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(2*GB, app_1.getHeadroom().getMemorySize());
- }
-
- @Test
public void testHeadroomWithMaxCap() throws Exception {
// Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
@@ -1403,7 +1315,12 @@ public class TestLeafQueue {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
+
+ ParentQueue root = (ParentQueue) queues
+ .get(CapacitySchedulerConfiguration.ROOT);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
@@ -1454,6 +1371,8 @@ public class TestLeafQueue {
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
priority, recordFactory)));
@@ -1542,6 +1461,8 @@ public class TestLeafQueue {
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getClusterResource()).thenReturn(clusterResource);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -1624,6 +1545,8 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -1638,6 +1561,8 @@ public class TestLeafQueue {
// Now, allocations should goto app_3 since it's under user-limit
a.setMaxCapacity(1.0f);
a.setUserLimitFactor(1);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -1743,6 +1668,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (4*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -1880,6 +1807,8 @@ public class TestLeafQueue {
final int numNodes = 3;
Resource clusterResource =
Resources.createResource(numNodes * (4*GB), numNodes * 16);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getMaximumResourceCapability()).thenReturn(
Resources.createResource(4*GB, 16));
@@ -2051,6 +1980,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2237,11 +2168,10 @@ public class TestLeafQueue {
CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
TestUtils.spyHook);
- queues = newQueues;
root.reinitialize(newRoot, cs.getClusterResource());
// Manipulate queue 'b'
- LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));
+ LeafQueue a = stubLeafQueue((LeafQueue) newQueues.get(B));
// Check locality parameters.
assertEquals(2, a.getNodeLocalityDelay());
@@ -2277,6 +2207,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8 * GB), numNodes * 16);
when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes);
+ newRoot.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2412,6 +2344,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>();
@@ -2545,6 +2479,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(
numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2660,17 +2596,14 @@ public class TestLeafQueue {
assertEquals(2, e.getNumActiveApplications());
assertEquals(1, e.getNumPendingApplications());
- csConf.setDouble(CapacitySchedulerConfiguration
- .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
- CapacitySchedulerConfiguration
- .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2);
+ csConf.setDouble(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
+ * 2);
Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
- CSQueue newRoot =
- CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
- CapacitySchedulerConfiguration.ROOT,
- newQueues, queues,
- TestUtils.spyHook);
- queues = newQueues;
+ CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+ csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+ TestUtils.spyHook);
root.reinitialize(newRoot, csContext.getClusterResource());
// after reinitialization
@@ -2697,7 +2630,6 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration.ROOT,
newQueues, queues,
TestUtils.spyHook);
- queues = newQueues;
root.reinitialize(newRoot, cs.getClusterResource());
// after reinitialization
@@ -2745,7 +2677,7 @@ public class TestLeafQueue {
assertEquals(1, e.getNumPendingApplications());
Resource clusterResource = Resources.createResource(200 * 16 * GB, 100 * 32);
- e.updateClusterResource(clusterResource,
+ root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
// after updating cluster resource
@@ -2837,6 +2769,9 @@ public class TestLeafQueue {
numNodes * (8*GB), numNodes * 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
// resourceName: <priority, memory, #containers, relaxLocality>
// host_0_0: < 1, 1GB, 1, true >
@@ -3036,36 +2971,44 @@ public class TestLeafQueue {
@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
throws Exception {
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- Resource clusterResource = Resources
- .createResource(100 * 16 * GB, 100 * 32);
+ final String newRootName = "root" + System.currentTimeMillis();
+ setupQueueConfiguration(csConf, newRootName);
+
+ Resource clusterResource = Resources.createResource(100 * 16 * GB,
+ 100 * 32);
CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
when(csContext.getRMContext()).thenReturn(rmContext);
- csConf.setFloat(CapacitySchedulerConfiguration.
- MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
- ParentQueue root = new ParentQueue(csContext,
- CapacitySchedulerConfiguration.ROOT, null, null);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
- LeafQueue a = new LeafQueue(csContext, A, root, null);
- assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(160 * GB, 1));
-
- csConf.setFloat(CapacitySchedulerConfiguration.
- MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
- LeafQueue newA = new LeafQueue(csContext, A, root, null);
- a.reinitialize(newA, clusterResource);
- assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(320 * GB, 1));
+ csConf.setFloat(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ 0.1f);
+
+ CSQueue root;
+ root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
- Resource newClusterResource = Resources.createResource(100 * 20 * GB,
- 100 * 32);
- a.updateClusterResource(newClusterResource,
- new ResourceLimits(newClusterResource));
- // 100 * 20 * 0.2 = 400
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(400 * GB, 1));
+ // Manipulate queue 'a'
+ LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B));
+ assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(b.calculateAndGetAMResourceLimit(),
+ Resources.createResource(159 * GB, 1));
+
+ csConf.setFloat(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ 0.2f);
+ clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32);
+ Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+ CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+ csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+ TestUtils.spyHook);
+ root.reinitialize(newRoot, clusterResource);
+
+ b = stubLeafQueue((LeafQueue) newQueues.get(B));
+ assertEquals(b.calculateAndGetAMResourceLimit(),
+ Resources.createResource(320 * GB, 1));
}
@Test
@@ -3142,6 +3085,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(numNodes * (16 * GB),
numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
String user_0 = "user_0";
@@ -3308,6 +3253,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(
numNodes * (16*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
String user_0 = "user_0";
@@ -3435,6 +3382,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
// App0 has node local request for host_0/host_1, and app1 has node local
@@ -3533,6 +3482,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (100*GB), numNodes * 128);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Pending resource requests for app_0 and app_1 total 5GB.
Priority priority = TestUtils.createMockPriority(1);
@@ -3699,6 +3650,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (100*GB), numNodes * 128);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Pending resource requests for user_0: app_0 and app_1 total 3GB.
Priority priority = TestUtils.createMockPriority(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index cdbbc51..a2318f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -248,6 +248,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
@@ -486,6 +488,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
@@ -695,6 +699,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
@@ -771,6 +777,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue b3 = (LeafQueue)queues.get(B3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 32f022f..0855720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -267,6 +267,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -454,6 +456,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -600,6 +604,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -782,6 +788,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -896,6 +904,10 @@ public class TestReservations {
String host_1 = "host_1";
FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
8 * GB);
+
+ Resource clusterResource = Resources.createResource(2 * 8 * GB);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
@@ -1068,6 +1080,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1256,6 +1270,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1418,6 +1434,9 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDFS-12547. Extend
TestQuotaWithStripedBlocks with a random EC policy. Contributed by Takanobu
Asanuma.
Posted by su...@apache.org.
HDFS-12547. Extend TestQuotaWithStripedBlocks with a random EC policy. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a297fb08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a297fb08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a297fb08
Branch: refs/heads/YARN-5881
Commit: a297fb08866305860dc17813c3db5701e9515101
Parents: 78af6cd
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Oct 10 17:35:49 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Oct 10 17:35:49 2017 -0700
----------------------------------------------------------------------
.../namenode/TestQuotaWithStripedBlocks.java | 40 ++++++++++------
...uotaWithStripedBlocksWithRandomECPolicy.java | 50 ++++++++++++++++++++
2 files changed, 76 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a297fb08/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
index 9995393..38b98a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
@@ -44,27 +44,39 @@ import java.io.IOException;
* Make sure we correctly update the quota usage with the striped blocks.
*/
public class TestQuotaWithStripedBlocks {
- private static final int BLOCK_SIZE = 1024 * 1024;
- private static final long DISK_QUOTA = BLOCK_SIZE * 10;
- private final ErasureCodingPolicy ecPolicy =
- StripedFileTestUtil.getDefaultECPolicy();
- private final int dataBlocks = ecPolicy.getNumDataUnits();
- private final int parityBlocsk = ecPolicy.getNumParityUnits();
- private final int groupSize = dataBlocks + parityBlocsk;
- private final int cellSize = ecPolicy.getCellSize();
- private static final Path ecDir = new Path("/ec");
+ private int blockSize;
+ private ErasureCodingPolicy ecPolicy;
+ private int dataBlocks;
+ private int parityBlocsk;
+ private int groupSize;
+ private int cellSize;
+ private Path ecDir;
+ private long diskQuota;
private MiniDFSCluster cluster;
private FSDirectory dir;
private DistributedFileSystem dfs;
+ public ErasureCodingPolicy getEcPolicy() {
+ return StripedFileTestUtil.getDefaultECPolicy();
+ }
+
@Rule
public Timeout globalTimeout = new Timeout(300000);
@Before
public void setUp() throws IOException {
+ blockSize = 1024 * 1024;
+ ecPolicy = getEcPolicy();
+ dataBlocks = ecPolicy.getNumDataUnits();
+ parityBlocsk = ecPolicy.getNumParityUnits();
+ groupSize = dataBlocks + parityBlocsk;
+ cellSize = ecPolicy.getCellSize();
+ ecDir = new Path("/ec");
+ diskQuota = blockSize * (groupSize + 1);
+
final Configuration conf = new Configuration();
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
cluster.waitActive();
@@ -75,8 +87,8 @@ public class TestQuotaWithStripedBlocks {
dfs.mkdirs(ecDir);
dfs.getClient()
.setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
- dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA);
- dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA);
+ dfs.setQuota(ecDir, Long.MAX_VALUE - 1, diskQuota);
+ dfs.setQuotaByStorageType(ecDir, StorageType.DISK, diskQuota);
dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
}
@@ -112,8 +124,8 @@ public class TestQuotaWithStripedBlocks {
final long diskUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.DISK);
// When we add a new block we update the quota using the full block size.
- Assert.assertEquals(BLOCK_SIZE * groupSize, spaceUsed);
- Assert.assertEquals(BLOCK_SIZE * groupSize, diskUsed);
+ Assert.assertEquals(blockSize * groupSize, spaceUsed);
+ Assert.assertEquals(blockSize * groupSize, diskUsed);
dfs.getClient().getNamenode().complete(file.toString(),
dfs.getClient().getClientName(), previous, fileNode.getId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a297fb08/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocksWithRandomECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocksWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocksWithRandomECPolicy.java
new file mode 100644
index 0000000..1828db1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocksWithRandomECPolicy.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test extends TestQuotaWithStripedBlocks to use a random
+ * (non-default) EC policy.
+ */
+public class TestQuotaWithStripedBlocksWithRandomECPolicy extends
+ TestQuotaWithStripedBlocks {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestQuotaWithStripedBlocksWithRandomECPolicy.class);
+
+ private ErasureCodingPolicy ecPolicy;
+
+ public TestQuotaWithStripedBlocksWithRandomECPolicy() {
+ // If you want to debug this test with a specific ec policy, please use
+ // SystemErasureCodingPolicies class.
+ // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
+ ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
+ LOG.info("run {} with {}.",
+ TestQuotaWithStripedBlocksWithRandomECPolicy.class
+ .getSuperclass().getSimpleName(), ecPolicy.getName());
+ }
+
+ @Override
+ public ErasureCodingPolicy getEcPolicy() {
+ return ecPolicy;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-6620. Add support in
NodeManager to isolate GPU devices by using CGroups. Contributed by Wangda
Tan.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
new file mode 100644
index 0000000..9576ce7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+
+public class GpuResourcePlugin implements ResourcePlugin {
+ private ResourceHandler gpuResourceHandler = null;
+ private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null;
+
+ @Override
+ public synchronized void initialize(Context context) throws YarnException {
+ resourceDiscoverHandler = new GpuNodeResourceUpdateHandler();
+ GpuDiscoverer.getInstance().initialize(context.getConf());
+ }
+
+ @Override
+ public synchronized ResourceHandler createResourceHandler(
+ Context context, CGroupsHandler cGroupsHandler,
+ PrivilegedOperationExecutor privilegedOperationExecutor) {
+ if (gpuResourceHandler == null) {
+ gpuResourceHandler = new GpuResourceHandlerImpl(context, cGroupsHandler,
+ privilegedOperationExecutor);
+ }
+
+ return gpuResourceHandler;
+ }
+
+ @Override
+ public synchronized NodeResourceUpdaterPlugin getNodeResourceHandlerInstance() {
+ return resourceDiscoverHandler;
+ }
+
+ @Override
+ public void cleanup() throws YarnException {
+ // Do nothing.
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
new file mode 100644
index 0000000..977032a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
+
+/**
+ * All GPU Device Information in the system.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@XmlRootElement(name = "nvidia_smi_log")
+public class GpuDeviceInformation {
+ List<PerGpuDeviceInformation> gpus;
+
+ String driverVersion = "N/A";
+
+ // More fields like topology information could be added when needed.
+ // ...
+
+ @javax.xml.bind.annotation.XmlElement(name = "gpu")
+ public List<PerGpuDeviceInformation> getGpus() {
+ return gpus;
+ }
+
+ public void setGpus(List<PerGpuDeviceInformation> gpus) {
+ this.gpus = gpus;
+ }
+
+ @javax.xml.bind.annotation.XmlElement(name = "driver_version")
+ public String getDriverVersion() {
+ return driverVersion;
+ }
+
+ public void setDriverVersion(String driverVersion) {
+ this.driverVersion = driverVersion;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("=== Gpus in the system ===\n").append("\tDriver Version:").append(
+ getDriverVersion()).append("\n");
+
+ if (gpus != null) {
+ for (PerGpuDeviceInformation gpu : gpus) {
+ sb.append("\t").append(gpu.toString()).append("\n");
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
new file mode 100644
index 0000000..1bd92f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.sax.SAXSource;
+import java.io.StringReader;
+
+/**
+ * Parse XML and get GPU device information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class GpuDeviceInformationParser {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ GpuDeviceInformationParser.class);
+
+ private Unmarshaller unmarshaller = null;
+ private XMLReader xmlReader = null;
+
+ private void init()
+ throws SAXException, ParserConfigurationException, JAXBException {
+ SAXParserFactory spf = SAXParserFactory.newInstance();
+ // Disable external-dtd since by default nvidia-smi output contains
+ // <!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd"> in header
+ spf.setFeature(
+ "http://apache.org/xml/features/nonvalidating/load-external-dtd",
+ false);
+ spf.setFeature("http://xml.org/sax/features/validation", false);
+
+ JAXBContext jaxbContext = JAXBContext.newInstance(
+ GpuDeviceInformation.class);
+
+ this.xmlReader = spf.newSAXParser().getXMLReader();
+ this.unmarshaller = jaxbContext.createUnmarshaller();
+ }
+
+ public synchronized GpuDeviceInformation parseXml(String xmlContent)
+ throws YarnException {
+ if (unmarshaller == null) {
+ try {
+ init();
+ } catch (SAXException | ParserConfigurationException | JAXBException e) {
+ LOG.error("Exception while initialize parser", e);
+ throw new YarnException(e);
+ }
+ }
+
+ InputSource inputSource = new InputSource(new StringReader(xmlContent));
+ SAXSource source = new SAXSource(xmlReader, inputSource);
+ try {
+ return (GpuDeviceInformation) unmarshaller.unmarshal(source);
+ } catch (JAXBException e) {
+ LOG.error("Exception while parsing xml", e);
+ throw new YarnException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
new file mode 100644
index 0000000..f315313
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlAdapter;
+
+/**
+ * Capture single GPU device information such as memory size, temperature,
+ * utilization.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@XmlRootElement(name = "gpu")
+public class PerGpuDeviceInformation {
+
+ private String productName = "N/A";
+ private String uuid = "N/A";
+ private int minorNumber = -1;
+
+ private PerGpuUtilizations gpuUtilizations;
+ private PerGpuMemoryUsage gpuMemoryUsage;
+ private PerGpuTemperature temperature;
+
+ /**
+ * Convert formats like "34 C", "75.6 %" to float.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ static class StrToFloatBeforeSpaceAdapter extends
+ XmlAdapter<String, Float> {
+ @Override
+ public String marshal(Float v) throws Exception {
+ if (v == null) {
+ return "";
+ }
+ return String.valueOf(v);
+ }
+
+ @Override
+ public Float unmarshal(String v) throws Exception {
+ if (v == null) {
+ return -1f;
+ }
+
+ return Float.valueOf(v.split(" ")[0]);
+ }
+ }
+
+ /**
+ * Convert formats like "725 MiB" to long.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ static class StrToMemAdapter extends XmlAdapter<String, Long> {
+ @Override
+ public String marshal(Long v) throws Exception {
+ if (v == null) {
+ return "";
+ }
+ return String.valueOf(v) + " MiB";
+ }
+
+ @Override
+ public Long unmarshal(String v) throws Exception {
+ if (v == null) {
+ return -1L;
+ }
+ return Long.valueOf(v.split(" ")[0]);
+ }
+ }
+
+ @XmlElement(name = "temperature")
+ public PerGpuTemperature getTemperature() {
+ return temperature;
+ }
+
+ public void setTemperature(PerGpuTemperature temperature) {
+ this.temperature = temperature;
+ }
+
+ @XmlElement(name = "uuid")
+ public String getUuid() {
+ return uuid;
+ }
+
+ public void setUuid(String uuid) {
+ this.uuid = uuid;
+ }
+
+ @XmlElement(name = "product_name")
+ public String getProductName() {
+ return productName;
+ }
+
+ public void setProductName(String productName) {
+ this.productName = productName;
+ }
+
+ @XmlElement(name = "minor_number")
+ public int getMinorNumber() {
+ return minorNumber;
+ }
+
+ public void setMinorNumber(int minorNumber) {
+ this.minorNumber = minorNumber;
+ }
+
+ @XmlElement(name = "utilization")
+ public PerGpuUtilizations getGpuUtilizations() {
+ return gpuUtilizations;
+ }
+
+ public void setGpuUtilizations(PerGpuUtilizations utilizations) {
+ this.gpuUtilizations = utilizations;
+ }
+
+ @XmlElement(name = "bar1_memory_usage")
+ public PerGpuMemoryUsage getGpuMemoryUsage() {
+ return gpuMemoryUsage;
+ }
+
+ public void setGpuMemoryUsage(PerGpuMemoryUsage gpuMemoryUsage) {
+ this.gpuMemoryUsage = gpuMemoryUsage;
+ }
+
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("ProductName=").append(productName).append(", MinorNumber=")
+ .append(minorNumber);
+
+ if (getGpuMemoryUsage() != null) {
+ sb.append(", TotalMemory=").append(
+ getGpuMemoryUsage().getTotalMemoryMiB()).append("MiB");
+ }
+
+ if (getGpuUtilizations() != null) {
+ sb.append(", Utilization=").append(
+ getGpuUtilizations().getOverallGpuUtilization()).append("%");
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
new file mode 100644
index 0000000..3964c4e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@XmlRootElement(name = "bar1_memory_usage")
+public class PerGpuMemoryUsage {
+ long usedMemoryMiB = -1L;
+ long availMemoryMiB = -1L;
+
+ @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToMemAdapter.class)
+ @XmlElement(name = "used")
+ public Long getUsedMemoryMiB() {
+ return usedMemoryMiB;
+ }
+
+ public void setUsedMemoryMiB(Long usedMemoryMiB) {
+ this.usedMemoryMiB = usedMemoryMiB;
+ }
+
+ @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToMemAdapter.class)
+ @XmlElement(name = "free")
+ public Long getAvailMemoryMiB() {
+ return availMemoryMiB;
+ }
+
+ public void setAvailMemoryMiB(Long availMemoryMiB) {
+ this.availMemoryMiB = availMemoryMiB;
+ }
+
+ public long getTotalMemoryMiB() {
+ return usedMemoryMiB + availMemoryMiB;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuTemperature.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuTemperature.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuTemperature.java
new file mode 100644
index 0000000..ccd60cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuTemperature.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+/**
+ * Temperature of GPU
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@XmlRootElement(name = "temperature")
+public class PerGpuTemperature {
+ private float currentGpuTemp = Float.MIN_VALUE;
+ private float maxGpuTemp = Float.MIN_VALUE;
+ private float slowThresholdGpuTemp = Float.MIN_VALUE;
+
+ /**
+ * Get current celsius GPU temperature
+ * @return temperature
+ */
+ @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToFloatBeforeSpaceAdapter.class)
+ @XmlElement(name = "gpu_temp")
+ public Float getCurrentGpuTemp() {
+ return currentGpuTemp;
+ }
+
+ public void setCurrentGpuTemp(Float currentGpuTemp) {
+ this.currentGpuTemp = currentGpuTemp;
+ }
+
+ /**
+ * Get max possible celsius GPU temperature
+ * @return temperature
+ */
+ @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToFloatBeforeSpaceAdapter.class)
+ @XmlElement(name = "gpu_temp_max_threshold")
+ public Float getMaxGpuTemp() {
+ return maxGpuTemp;
+ }
+
+ public void setMaxGpuTemp(Float maxGpuTemp) {
+ this.maxGpuTemp = maxGpuTemp;
+ }
+
+ /**
+ * Get celsius GPU temperature which could make GPU runs slower
+ * @return temperature
+ */
+ @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToFloatBeforeSpaceAdapter.class)
+ @XmlElement(name = "gpu_temp_slow_threshold")
+ public Float getSlowThresholdGpuTemp() {
+ return slowThresholdGpuTemp;
+ }
+
+ public void setSlowThresholdGpuTemp(Float slowThresholdGpuTemp) {
+ this.slowThresholdGpuTemp = slowThresholdGpuTemp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuUtilizations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuUtilizations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuUtilizations.java
new file mode 100644
index 0000000..4ef218b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuUtilizations.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+/**
+ * GPU utilizations
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@XmlRootElement(name = "utilization")
+public class PerGpuUtilizations {
+ private float overallGpuUtilization;
+
+ /**
+ * Overall percent GPU utilization
+ * @return utilization
+ */
+ @XmlJavaTypeAdapter(PerGpuDeviceInformation.StrToFloatBeforeSpaceAdapter.class)
+ @XmlElement(name = "gpu_util")
+ public Float getOverallGpuUtilization() {
+ return overallGpuUtilization;
+ }
+
+ public void setOverallGpuUtilization(Float overallGpuUtilization) {
+ this.overallGpuUtilization = overallGpuUtilization;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerTestBase.java
new file mode 100644
index 0000000..13b3ee9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerTestBase.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.ServerSocketUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.api.ResourceTracker;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterNodeManagerResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UnRegisterNodeManagerResponsePBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.junit.Assert;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+public class NodeManagerTestBase {
+ // temp fix until metrics system can auto-detect itself running in unit test:
+ static {
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ }
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestNodeStatusUpdater.class);
+ protected static final File basedir =
+ new File("target", TestNodeStatusUpdater.class.getName());
+ protected static final File nmLocalDir = new File(basedir, "nm0");
+ protected static final File tmpDir = new File(basedir, "tmpDir");
+ protected static final File remoteLogsDir = new File(basedir, "remotelogs");
+ protected static final File logsDir = new File(basedir, "logs");
+ protected static final RecordFactory recordFactory = RecordFactoryProvider
+ .getRecordFactory(null);
+ protected Configuration conf;
+
+ protected YarnConfiguration createNMConfig() throws IOException {
+ return createNMConfig(ServerSocketUtil.getPort(49170, 10));
+ }
+
+ protected YarnConfiguration createNMConfig(int port) throws IOException {
+ YarnConfiguration conf = new YarnConfiguration();
+ String localhostAddress = null;
+ try {
+ localhostAddress = InetAddress.getByName("localhost")
+ .getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ Assert.fail("Unable to get localhost address: " + e.getMessage());
+ }
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
+ conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":" + port);
+ conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":"
+ + ServerSocketUtil.getPort(49160, 10));
+ conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ remoteLogsDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
+ conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
+ return conf;
+ }
+
+ public static class BaseResourceTrackerForTest implements ResourceTracker {
+ @Override
+ public RegisterNodeManagerResponse registerNodeManager(
+ RegisterNodeManagerRequest request) throws YarnException, IOException {
+ return new RegisterNodeManagerResponsePBImpl();
+ }
+
+ @Override
+ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
+ throws YarnException, IOException {
+ return new NodeHeartbeatResponsePBImpl();
+ }
+
+ @Override
+ public UnRegisterNodeManagerResponse unRegisterNodeManager(
+ UnRegisterNodeManagerRequest request)
+ throws YarnException, IOException {
+ return new UnRegisterNodeManagerResponsePBImpl();
+ }
+ }
+
+ protected static class BaseNodeStatusUpdaterForTest extends NodeStatusUpdaterImpl {
+ public ResourceTracker resourceTracker;
+ protected Context context;
+
+ public BaseNodeStatusUpdaterForTest(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
+ ResourceTracker resourceTracker) {
+ super(context, dispatcher, healthChecker, metrics);
+ this.context = context;
+ this.resourceTracker = resourceTracker;
+ }
+ @Override
+ protected ResourceTracker getRMClient() {
+ return resourceTracker;
+ }
+
+ @Override
+ protected void stopRMProxy() {
+ return;
+ }
+ }
+
+ public class MyContainerManager extends ContainerManagerImpl {
+ public boolean signaled = false;
+
+ public MyContainerManager(Context context, ContainerExecutor exec,
+ DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler) {
+ super(context, exec, deletionContext, nodeStatusUpdater,
+ metrics, dirsHandler);
+ }
+
+ @Override
+ public void handle(ContainerManagerEvent event) {
+ if (event.getType() == ContainerManagerEventType.SIGNAL_CONTAINERS) {
+ signaled = true;
+ }
+ }
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ nmLocalDir.mkdirs();
+ tmpDir.mkdirs();
+ logsDir.mkdirs();
+ remoteLogsDir.mkdirs();
+ conf = createNMConfig();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
index 2e9eff5..9b180c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
@@ -178,7 +178,7 @@ public class TestDefaultContainerExecutor {
FileContext lfs = FileContext.getLocalFSFileContext(conf);
DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);
executor.setConf(conf);
- executor.init();
+ executor.init(null);
try {
executor.createUserLocalDirs(localDirs, user);
@@ -317,7 +317,7 @@ public class TestDefaultContainerExecutor {
Path workDir = localDir;
Path pidFile = new Path(workDir, "pid.txt");
- mockExec.init();
+ mockExec.init(null);
mockExec.activateContainer(cId, pidFile);
int ret = mockExec.launchContainer(new ContainerStartContext.Builder()
.setContainer(container)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index d4db6b0..dcec4c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -628,7 +628,7 @@ public class TestLinuxContainerExecutor {
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.setConf(conf);
try {
- lce.init();
+ lce.init(null);
} catch (IOException e) {
// expected if LCE isn't setup right, but not necessary for this test
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index ad675b1..3dfa625 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -415,7 +415,7 @@ public class TestLinuxContainerExecutorWithMocks {
@Test
public void testInit() throws Exception {
- mockExec.init();
+ mockExec.init(mock(Context.class));
assertEquals(Arrays.asList("--checksetup"), readMockParams());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
index 9279711..b31215b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
@@ -37,7 +37,7 @@ public class TestNodeManager {
public static final class InvalidContainerExecutor extends
DefaultContainerExecutor {
@Override
- public void init() throws IOException {
+ public void init(Context nmContext) throws IOException {
throw new IOException("dummy executor init called");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 11c3c35..8435340 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -20,16 +20,14 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils.newNodeHeartbeatResponse;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -81,8 +79,6 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
@@ -118,41 +114,14 @@ import org.junit.Before;
import org.junit.Test;
@SuppressWarnings("rawtypes")
-public class TestNodeStatusUpdater {
-
- // temp fix until metrics system can auto-detect itself running in unit test:
- static {
- DefaultMetricsSystem.setMiniClusterMode(true);
- }
-
- static final Logger LOG =
- LoggerFactory.getLogger(TestNodeStatusUpdater.class);
- static final File basedir =
- new File("target", TestNodeStatusUpdater.class.getName());
- static final File nmLocalDir = new File(basedir, "nm0");
- static final File tmpDir = new File(basedir, "tmpDir");
- static final File remoteLogsDir = new File(basedir, "remotelogs");
- static final File logsDir = new File(basedir, "logs");
- private static final RecordFactory recordFactory = RecordFactoryProvider
- .getRecordFactory(null);
-
+public class TestNodeStatusUpdater extends NodeManagerTestBase {
volatile int heartBeatID = 0;
volatile Throwable nmStartError = null;
private final List<NodeId> registeredNodes = new ArrayList<NodeId>();
private boolean triggered = false;
- private Configuration conf;
private NodeManager nm;
private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
- @Before
- public void setUp() throws IOException {
- nmLocalDir.mkdirs();
- tmpDir.mkdirs();
- logsDir.mkdirs();
- remoteLogsDir.mkdirs();
- conf = createNMConfig();
- }
-
@After
public void tearDown() {
this.registeredNodes.clear();
@@ -334,29 +303,7 @@ public class TestNodeStatusUpdater {
}
}
- private class MyContainerManager extends ContainerManagerImpl {
- public boolean signaled = false;
-
- public MyContainerManager(Context context, ContainerExecutor exec,
- DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics,
- LocalDirsHandlerService dirsHandler) {
- super(context, exec, deletionContext, nodeStatusUpdater,
- metrics, dirsHandler);
- }
-
- @Override
- public void handle(ContainerManagerEvent event) {
- if (event.getType() == ContainerManagerEventType.SIGNAL_CONTAINERS) {
- signaled = true;
- }
- }
- }
-
- private class MyNodeStatusUpdater extends NodeStatusUpdaterImpl {
- public ResourceTracker resourceTracker;
- private Context context;
-
+ private class MyNodeStatusUpdater extends BaseNodeStatusUpdaterForTest {
public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
this(context, dispatcher, healthChecker, metrics, false);
@@ -365,19 +312,8 @@ public class TestNodeStatusUpdater {
public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
boolean signalContainer) {
- super(context, dispatcher, healthChecker, metrics);
- this.context = context;
- resourceTracker = new MyResourceTracker(this.context, signalContainer);
- }
-
- @Override
- protected ResourceTracker getRMClient() {
- return resourceTracker;
- }
-
- @Override
- protected void stopRMProxy() {
- return;
+ super(context, dispatcher, healthChecker, metrics,
+ new MyResourceTracker(context, signalContainer));
}
}
@@ -1820,7 +1756,6 @@ public class TestNodeStatusUpdater {
Assert.assertTrue("Test failed with exception(s)" + exceptions,
exceptions.isEmpty());
}
-
// Add new containers info into NM context each time node heart beats.
private class MyNMContext extends NMContext {
@@ -1924,31 +1859,6 @@ public class TestNodeStatusUpdater {
this.registeredNodes.size());
}
- private YarnConfiguration createNMConfig(int port) throws IOException {
- YarnConfiguration conf = new YarnConfiguration();
- String localhostAddress = null;
- try {
- localhostAddress = InetAddress.getByName("localhost")
- .getCanonicalHostName();
- } catch (UnknownHostException e) {
- Assert.fail("Unable to get localhost address: " + e.getMessage());
- }
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
- conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":" + port);
- conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":"
- + ServerSocketUtil.getPort(49160, 10));
- conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
- conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
- remoteLogsDir.getAbsolutePath());
- conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
- conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
- return conf;
- }
-
- private YarnConfiguration createNMConfig() throws IOException {
- return createNMConfig(ServerSocketUtil.getPort(49170, 10));
- }
-
private NodeManager getNodeManager(final NodeAction nodeHeartBeatAction) {
return new NodeManager() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 0838f1e..3c57496 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -18,26 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -64,6 +44,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -72,17 +53,36 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
-import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
+import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
/**
* Base class for all the AMRMProxyService test cases. It provides utility
@@ -773,5 +773,9 @@ public abstract class BaseAMRMProxyTest {
getContainerStateTransitionListener() {
return null;
}
+
+ public ResourcePluginManager getResourcePluginManager() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
index e5414a5..0563694 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
import java.util.List;
+import static org.mockito.Mockito.mock;
+
public class TestResourceHandlerModule {
private static final Logger LOG =
LoggerFactory.getLogger(TestResourceHandlerModule.class);
@@ -62,7 +65,7 @@ public class TestResourceHandlerModule {
//Ensure that outbound bandwidth resource handler is present in the chain
ResourceHandlerChain resourceHandlerChain = ResourceHandlerModule
- .getConfiguredResourceHandlerChain(networkEnabledConf);
+ .getConfiguredResourceHandlerChain(networkEnabledConf, mock(Context.class));
List<ResourceHandler> resourceHandlers = resourceHandlerChain
.getResourceHandlerList();
//Exactly one resource handler in chain
@@ -88,7 +91,8 @@ public class TestResourceHandlerModule {
Assert.assertNotNull(handler);
ResourceHandlerChain resourceHandlerChain =
- ResourceHandlerModule.getConfiguredResourceHandlerChain(diskConf);
+ ResourceHandlerModule.getConfiguredResourceHandlerChain(diskConf,
+ mock(Context.class));
List<ResourceHandler> resourceHandlers =
resourceHandlerChain.getResourceHandlerList();
// Exactly one resource handler in chain
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
new file mode 100644
index 0000000..5c70f7a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -0,0 +1,382 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestGpuResourceHandler {
+ private CGroupsHandler mockCGroupsHandler;
+ private PrivilegedOperationExecutor mockPrivilegedExecutor;
+ private GpuResourceHandlerImpl gpuResourceHandler;
+ private NMStateStoreService mockNMStateStore;
+ private ConcurrentHashMap<ContainerId, Container> runningContainersMap;
+
+ @Before
+ public void setup() {
+ TestResourceUtils.addNewTypesToResources(ResourceInformation.GPU_URI);
+
+ mockCGroupsHandler = mock(CGroupsHandler.class);
+ mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
+ mockNMStateStore = mock(NMStateStoreService.class);
+
+ Context nmctx = mock(Context.class);
+ when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore);
+ runningContainersMap = new ConcurrentHashMap<>();
+ when(nmctx.getContainers()).thenReturn(runningContainersMap);
+
+ gpuResourceHandler = new GpuResourceHandlerImpl(nmctx, mockCGroupsHandler,
+ mockPrivilegedExecutor);
+ }
+
+ @Test
+ public void testBootStrap() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0");
+
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuResourceHandler.bootstrap(conf);
+ verify(mockCGroupsHandler, times(1)).initializeCGroupController(
+ CGroupsHandler.CGroupController.DEVICES);
+ }
+
+ private static ContainerId getContainerId(int id) {
+ return ContainerId.newContainerId(ApplicationAttemptId
+ .newInstance(ApplicationId.newInstance(1234L, 1), 1), id);
+ }
+
+ private static Container mockContainerWithGpuRequest(int id,
+ int numGpuRequest) {
+ Container c = mock(Container.class);
+ when(c.getContainerId()).thenReturn(getContainerId(id));
+
+ Resource res = Resource.newInstance(1024, 1);
+ ResourceMappings resMapping = new ResourceMappings();
+
+ res.setResourceValue(ResourceInformation.GPU_URI, numGpuRequest);
+ when(c.getResource()).thenReturn(res);
+ when(c.getResourceMappings()).thenReturn(resMapping);
+ return c;
+ }
+
+ private void verifyDeniedDevices(ContainerId containerId,
+ List<Integer> deniedDevices)
+ throws ResourceHandlerException, PrivilegedOperationException {
+ verify(mockCGroupsHandler, times(1)).createCGroup(
+ CGroupsHandler.CGroupController.DEVICES, containerId.toString());
+
+ if (null != deniedDevices && !deniedDevices.isEmpty()) {
+ verify(mockPrivilegedExecutor, times(1)).executePrivilegedOperation(
+ new PrivilegedOperation(PrivilegedOperation.OperationType.GPU, Arrays
+ .asList(GpuResourceHandlerImpl.CONTAINER_ID_CLI_OPTION,
+ containerId.toString(),
+ GpuResourceHandlerImpl.EXCLUDED_GPUS_CLI_OPTION,
+ StringUtils.join(",", deniedDevices))), true);
+ }
+ }
+
+ @Test
+ public void testAllocation() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuResourceHandler.bootstrap(conf);
+ Assert.assertEquals(4,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ /* Start container 1, asks 3 containers */
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(1, 3));
+
+ // Only device=4 will be blocked.
+ verifyDeniedDevices(getContainerId(1), Arrays.asList(4));
+
+ /* Start container 2, asks 2 containers. Excepted to fail */
+ boolean failedToAllocate = false;
+ try {
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(2, 2));
+ } catch (ResourceHandlerException e) {
+ failedToAllocate = true;
+ }
+ Assert.assertTrue(failedToAllocate);
+
+ /* Start container 3, ask 1 container, succeeded */
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(3, 1));
+
+ // devices = 0/1/3 will be blocked
+ verifyDeniedDevices(getContainerId(3), Arrays.asList(0, 1, 3));
+
+ /* Start container 4, ask 0 container, succeeded */
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(4, 0));
+
+ // All devices will be blocked
+ verifyDeniedDevices(getContainerId(4), Arrays.asList(0, 1, 3, 4));
+
+ /* Release container-1, expect cgroups deleted */
+ gpuResourceHandler.postComplete(getContainerId(1));
+
+ verify(mockCGroupsHandler, times(1)).createCGroup(
+ CGroupsHandler.CGroupController.DEVICES, getContainerId(1).toString());
+ Assert.assertEquals(3,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ /* Release container-3, expect cgroups deleted */
+ gpuResourceHandler.postComplete(getContainerId(3));
+
+ verify(mockCGroupsHandler, times(1)).createCGroup(
+ CGroupsHandler.CGroupController.DEVICES, getContainerId(3).toString());
+ Assert.assertEquals(4,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testAssignedGpuWillBeCleanedupWhenStoreOpFails()
+ throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuResourceHandler.bootstrap(conf);
+ Assert.assertEquals(4,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ doThrow(new IOException("Exception ...")).when(mockNMStateStore)
+ .storeAssignedResources(
+ any(ContainerId.class), anyString(), anyList());
+
+ boolean exception = false;
+ /* Start container 1, asks 3 containers */
+ try {
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(1, 3));
+ } catch (ResourceHandlerException e) {
+ exception = true;
+ }
+
+ Assert.assertTrue("preStart should throw exception", exception);
+
+ // After preStart, we still have 4 available GPU since the store op fails.
+ Assert.assertEquals(4,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+ }
+
+ @Test
+ public void testAllocationWithoutAllowedGpus() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, " ");
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuResourceHandler.bootstrap(conf);
+ Assert.assertEquals(0,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ /* Start container 1, asks 0 containers */
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(1, 0));
+ verifyDeniedDevices(getContainerId(1), Collections.emptyList());
+
+ /* Start container 2, asks 1 containers. Excepted to fail */
+ boolean failedToAllocate = false;
+ try {
+ gpuResourceHandler.preStart(mockContainerWithGpuRequest(2, 1));
+ } catch (ResourceHandlerException e) {
+ failedToAllocate = true;
+ }
+ Assert.assertTrue(failedToAllocate);
+
+ /* Release container 1, expect cgroups deleted */
+ gpuResourceHandler.postComplete(getContainerId(1));
+
+ verify(mockCGroupsHandler, times(1)).createCGroup(
+ CGroupsHandler.CGroupController.DEVICES, getContainerId(1).toString());
+ Assert.assertEquals(0,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+ }
+
+ @Test
+ public void testAllocationStored() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuResourceHandler.bootstrap(conf);
+ Assert.assertEquals(4,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ /* Start container 1, asks 3 containers */
+ Container container = mockContainerWithGpuRequest(1, 3);
+ gpuResourceHandler.preStart(container);
+
+ verify(mockNMStateStore).storeAssignedResources(getContainerId(1),
+ ResourceInformation.GPU_URI,
+ Arrays.asList("0", "1", "3"));
+
+ Assert.assertEquals(3, container.getResourceMappings()
+ .getAssignedResources(ResourceInformation.GPU_URI).size());
+
+ // Only device=4 will be blocked.
+ verifyDeniedDevices(getContainerId(1), Arrays.asList(4));
+
+ /* Start container 2, ask 0 container, succeeded */
+ container = mockContainerWithGpuRequest(2, 0);
+ gpuResourceHandler.preStart(container);
+
+ verifyDeniedDevices(getContainerId(2), Arrays.asList(0, 1, 3, 4));
+ Assert.assertEquals(0, container.getResourceMappings()
+ .getAssignedResources(ResourceInformation.GPU_URI).size());
+
+ // Store assigned resource will not be invoked.
+ verify(mockNMStateStore, never()).storeAssignedResources(
+ eq(getContainerId(2)), eq(ResourceInformation.GPU_URI), anyList());
+ }
+
+ @Test
+ public void testRecoverResourceAllocation() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuResourceHandler.bootstrap(conf);
+ Assert.assertEquals(4,
+ gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ Container nmContainer = mock(Container.class);
+ ResourceMappings rmap = new ResourceMappings();
+ ResourceMappings.AssignedResources ar =
+ new ResourceMappings.AssignedResources();
+ ar.updateAssignedResources(Arrays.asList("1", "3"));
+ rmap.addAssignedResources(ResourceInformation.GPU_URI, ar);
+ when(nmContainer.getResourceMappings()).thenReturn(rmap);
+
+ runningContainersMap.put(getContainerId(1), nmContainer);
+
+ // TEST CASE
+ // Reacquire container restore state of GPU Resource Allocator.
+ gpuResourceHandler.reacquireContainer(getContainerId(1));
+
+ Map<Integer, ContainerId> deviceAllocationMapping =
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ Assert.assertEquals(2, deviceAllocationMapping.size());
+ Assert.assertTrue(
+ deviceAllocationMapping.keySet().containsAll(Arrays.asList(1, 3)));
+ Assert.assertEquals(deviceAllocationMapping.get(1), getContainerId(1));
+
+ // TEST CASE
+ // Try to reacquire a container but requested device is not in allowed list.
+ nmContainer = mock(Container.class);
+ rmap = new ResourceMappings();
+ ar = new ResourceMappings.AssignedResources();
+ // id=5 is not in allowed list.
+ ar.updateAssignedResources(Arrays.asList("4", "5"));
+ rmap.addAssignedResources(ResourceInformation.GPU_URI, ar);
+ when(nmContainer.getResourceMappings()).thenReturn(rmap);
+
+ runningContainersMap.put(getContainerId(2), nmContainer);
+
+ boolean caughtException = false;
+ try {
+ gpuResourceHandler.reacquireContainer(getContainerId(1));
+ } catch (ResourceHandlerException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue(
+ "Should fail since requested device Id is not in allowed list",
+ caughtException);
+
+ // Make sure internal state not changed.
+ deviceAllocationMapping =
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ Assert.assertEquals(2, deviceAllocationMapping.size());
+ Assert.assertTrue(
+ deviceAllocationMapping.keySet().containsAll(Arrays.asList(1, 3)));
+ Assert.assertEquals(deviceAllocationMapping.get(1), getContainerId(1));
+
+ // TEST CASE
+ // Try to reacquire a container but requested device is already assigned.
+ nmContainer = mock(Container.class);
+ rmap = new ResourceMappings();
+ ar = new ResourceMappings.AssignedResources();
+ // id=3 is already assigned
+ ar.updateAssignedResources(Arrays.asList("4", "3"));
+ rmap.addAssignedResources("gpu", ar);
+ when(nmContainer.getResourceMappings()).thenReturn(rmap);
+
+ runningContainersMap.put(getContainerId(2), nmContainer);
+
+ caughtException = false;
+ try {
+ gpuResourceHandler.reacquireContainer(getContainerId(1));
+ } catch (ResourceHandlerException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue(
+ "Should fail since requested device Id is not in allowed list",
+ caughtException);
+
+ // Make sure internal state not changed.
+ deviceAllocationMapping =
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ Assert.assertEquals(2, deviceAllocationMapping.size());
+ Assert.assertTrue(
+ deviceAllocationMapping.keySet().containsAll(Arrays.asList(1, 3)));
+ Assert.assertEquals(deviceAllocationMapping.get(1), getContainerId(1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index e21eea0..2cca277 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -73,7 +73,7 @@ public class TestContainersMonitorResourceChange {
private static class MockExecutor extends ContainerExecutor {
@Override
- public void init() throws IOException {
+ public void init(Context nmContext) throws IOException {
}
@Override
public void startLocalizer(LocalizerStartContext ctx)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
new file mode 100644
index 0000000..bcadf76
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManagerTestBase;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestResourcePluginManager extends NodeManagerTestBase {
+ private NodeManager nm;
+
+ ResourcePluginManager stubResourcePluginmanager() {
+ // Stub ResourcePluginManager
+ final ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ Map<String, ResourcePlugin> plugins = new HashMap<>();
+
+ // First resource plugin
+ ResourcePlugin resourcePlugin = mock(ResourcePlugin.class);
+ NodeResourceUpdaterPlugin nodeResourceUpdaterPlugin = mock(
+ NodeResourceUpdaterPlugin.class);
+ when(resourcePlugin.getNodeResourceHandlerInstance()).thenReturn(
+ nodeResourceUpdaterPlugin);
+ plugins.put("resource1", resourcePlugin);
+
+ // Second resource plugin
+ resourcePlugin = mock(ResourcePlugin.class);
+ when(resourcePlugin.createResourceHandler(any(Context.class), any(
+ CGroupsHandler.class), any(PrivilegedOperationExecutor.class)))
+ .thenReturn(new CustomizedResourceHandler());
+ plugins.put("resource2", resourcePlugin);
+ when(rpm.getNameToPlugins()).thenReturn(plugins);
+ return rpm;
+ }
+
+ @After
+ public void tearDown() {
+ if (nm != null) {
+ try {
+ ServiceOperations.stop(nm);
+ } catch (Throwable t) {
+ // ignore
+ }
+ }
+ }
+
+ private class CustomizedResourceHandler implements ResourceHandler {
+
+ @Override
+ public List<PrivilegedOperation> bootstrap(Configuration configuration)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> preStart(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> reacquireContainer(ContainerId containerId)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> postComplete(ContainerId containerId)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> teardown()
+ throws ResourceHandlerException {
+ return null;
+ }
+ }
+
+ private class MyMockNM extends NodeManager {
+ private final ResourcePluginManager rpm;
+
+ public MyMockNM(ResourcePluginManager rpm) {
+ this.rpm = rpm;
+ }
+
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ ((NodeManager.NMContext)context).setResourcePluginManager(rpm);
+ return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
+ metrics, new BaseResourceTrackerForTest());
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(Context context,
+ ContainerExecutor exec, DeletionService del,
+ NodeStatusUpdater nodeStatusUpdater,
+ ApplicationACLsManager aclsManager,
+ LocalDirsHandlerService diskhandler) {
+ return new MyContainerManager(context, exec, del, nodeStatusUpdater,
+ metrics, diskhandler);
+ }
+
+ @Override
+ protected ResourcePluginManager createResourcePluginManager() {
+ return rpm;
+ }
+ }
+
+ public class MyLCE extends LinuxContainerExecutor {
+ private PrivilegedOperationExecutor poe = mock(PrivilegedOperationExecutor.class);
+
+ @Override
+ protected PrivilegedOperationExecutor getPrivilegedOperationExecutor() {
+ return poe;
+ }
+ }
+
+ /*
+ * Make sure ResourcePluginManager is initialized during NM start up.
+ */
+ @Test(timeout = 30000)
+ public void testResourcePluginManagerInitialization() throws Exception {
+ final ResourcePluginManager rpm = stubResourcePluginmanager();
+ nm = new MyMockNM(rpm);
+
+ YarnConfiguration conf = createNMConfig();
+ nm.init(conf);
+ verify(rpm, times(1)).initialize(
+ any(Context.class));
+ }
+
+ /*
+ * Make sure ResourcePluginManager is invoked during NM update.
+ */
+ @Test(timeout = 30000)
+ public void testNodeStatusUpdaterWithResourcePluginsEnabled() throws Exception {
+ final ResourcePluginManager rpm = stubResourcePluginmanager();
+
+ nm = new MyMockNM(rpm);
+
+ YarnConfiguration conf = createNMConfig();
+ nm.init(conf);
+ nm.start();
+
+ NodeResourceUpdaterPlugin nodeResourceUpdaterPlugin =
+ rpm.getNameToPlugins().get("resource1")
+ .getNodeResourceHandlerInstance();
+
+ verify(nodeResourceUpdaterPlugin, times(1)).updateConfiguredResource(
+ any(Resource.class));
+ }
+
+ /*
+ * Make sure ResourcePluginManager is used to initialize ResourceHandlerChain
+ */
+ @Test(timeout = 30000)
+ public void testLinuxContainerExecutorWithResourcePluginsEnabled() throws Exception {
+ final ResourcePluginManager rpm = stubResourcePluginmanager();
+ final LinuxContainerExecutor lce = new MyLCE();
+
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ ((NMContext)context).setResourcePluginManager(rpm);
+ return new BaseNodeStatusUpdaterForTest(context, dispatcher, healthChecker,
+ metrics, new BaseResourceTrackerForTest());
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(Context context,
+ ContainerExecutor exec, DeletionService del,
+ NodeStatusUpdater nodeStatusUpdater,
+ ApplicationACLsManager aclsManager,
+ LocalDirsHandlerService diskhandler) {
+ return new MyContainerManager(context, exec, del, nodeStatusUpdater,
+ metrics, diskhandler);
+ }
+
+ @Override
+ protected ContainerExecutor createContainerExecutor(Configuration conf) {
+ ((NMContext)this.getNMContext()).setResourcePluginManager(rpm);
+ lce.setConf(conf);
+ return lce;
+ }
+ };
+
+ YarnConfiguration conf = createNMConfig();
+
+ nm.init(conf);
+ nm.start();
+
+ ResourceHandler handler = lce.getResourceHandler();
+ Assert.assertNotNull(handler);
+ Assert.assertTrue(handler instanceof ResourceHandlerChain);
+
+ boolean newHandlerAdded = false;
+ for (ResourceHandler h : ((ResourceHandlerChain) handler)
+ .getResourceHandlerList()) {
+ if (h instanceof CustomizedResourceHandler) {
+ newHandlerAdded = true;
+ break;
+ }
+ }
+ Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java
new file mode 100644
index 0000000..83bace2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+public class TestGpuDiscoverer {
+ private String getTestParentFolder() {
+ File f = new File("target/temp/" + TestGpuDiscoverer.class.getName());
+ return f.getAbsolutePath();
+ }
+
+ private void touchFile(File f) throws IOException {
+ new FileOutputStream(f).close();
+ }
+
+ @Before
+ public void before() throws IOException {
+ String folder = getTestParentFolder();
+ File f = new File(folder);
+ FileUtils.deleteDirectory(f);
+ f.mkdirs();
+ }
+
+ @Test
+ public void testLinuxGpuResourceDiscoverPluginConfig() throws Exception {
+ // Only run this on demand.
+ Assume.assumeTrue(Boolean.valueOf(
+ System.getProperty("RunLinuxGpuResourceDiscoverPluginConfigTest")));
+
+ // test case 1, check default setting.
+ Configuration conf = new Configuration(false);
+ GpuDiscoverer plugin = new GpuDiscoverer();
+ plugin.initialize(conf);
+ Assert.assertEquals(GpuDiscoverer.DEFAULT_BINARY_NAME,
+ plugin.getPathOfGpuBinary());
+ Assert.assertNotNull(plugin.getEnvironmentToRunCommand().get("PATH"));
+ Assert.assertTrue(
+ plugin.getEnvironmentToRunCommand().get("PATH").contains("nvidia"));
+
+ // test case 2, check mandatory set path.
+ File fakeBinary = new File(getTestParentFolder(),
+ GpuDiscoverer.DEFAULT_BINARY_NAME);
+ touchFile(fakeBinary);
+ conf.set(YarnConfiguration.NM_GPU_PATH_TO_EXEC, getTestParentFolder());
+ plugin = new GpuDiscoverer();
+ plugin.initialize(conf);
+ Assert.assertEquals(fakeBinary.getAbsolutePath(),
+ plugin.getPathOfGpuBinary());
+ Assert.assertNull(plugin.getEnvironmentToRunCommand().get("PATH"));
+
+ // test case 3, check mandatory set path, but binary doesn't exist so default
+ // path will be used.
+ fakeBinary.delete();
+ plugin = new GpuDiscoverer();
+ plugin.initialize(conf);
+ Assert.assertEquals(GpuDiscoverer.DEFAULT_BINARY_NAME,
+ plugin.getPathOfGpuBinary());
+ Assert.assertTrue(
+ plugin.getEnvironmentToRunCommand().get("PATH").contains("nvidia"));
+ }
+
+ @Test
+ public void testGpuDiscover() throws YarnException {
+ // Since this is more of a performance unit test, only run if
+ // RunUserLimitThroughput is set (-DRunUserLimitThroughput=true)
+ Assume.assumeTrue(
+ Boolean.valueOf(System.getProperty("runGpuDiscoverUnitTest")));
+ Configuration conf = new Configuration(false);
+ GpuDiscoverer plugin = new GpuDiscoverer();
+ plugin.initialize(conf);
+ GpuDeviceInformation info = plugin.getGpuDeviceInformation();
+
+ Assert.assertTrue(info.getGpus().size() > 0);
+ Assert.assertEquals(plugin.getMinorNumbersOfGpusUsableByYarn().size(),
+ info.getGpus().size());
+ }
+
+ @Test
+ public void getNumberOfUsableGpusFromConfig() throws YarnException {
+ Configuration conf = new Configuration(false);
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,2,4");
+ GpuDiscoverer plugin = new GpuDiscoverer();
+ plugin.initialize(conf);
+
+ List<Integer> minorNumbers = plugin.getMinorNumbersOfGpusUsableByYarn();
+ Assert.assertEquals(4, minorNumbers.size());
+
+ Assert.assertTrue(0 == minorNumbers.get(0));
+ Assert.assertTrue(1 == minorNumbers.get(1));
+ Assert.assertTrue(2 == minorNumbers.get(2));
+ Assert.assertTrue(4 == minorNumbers.get(3));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)
Change-Id: I9213f5297a6841fab5c573e85ee4c4e5f4a0b7ff
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cc00b7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cc00b7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cc00b7f
Branch: refs/heads/YARN-5881
Commit: 8cc00b7f7b6ed4fab55b87e742c74177cd70bf39
Parents: 075358e
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 11 10:30:23 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 12 15:05:55 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/StringUtils.java | 31 ++
.../resource/DefaultResourceCalculator.java | 6 +
.../resource/DominantResourceCalculator.java | 7 +
.../yarn/util/resource/ResourceCalculator.java | 12 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../capacity/FifoCandidatesSelector.java | 9 +-
.../ProportionalCapacityPreemptionPolicy.java | 10 +-
.../monitor/capacity/TempQueuePerPartition.java | 16 +-
.../scheduler/AbstractResourceUsage.java | 198 +++++++
.../scheduler/QueueResourceQuotas.java | 153 ++++++
.../scheduler/ResourceUsage.java | 237 ++-------
.../scheduler/capacity/AbstractCSQueue.java | 162 +++++-
.../scheduler/capacity/CSQueue.java | 42 +-
.../scheduler/capacity/CSQueueUtils.java | 24 +-
.../CapacitySchedulerConfiguration.java | 179 ++++++-
.../scheduler/capacity/LeafQueue.java | 31 +-
.../scheduler/capacity/ParentQueue.java | 201 +++++++-
.../scheduler/capacity/UsersManager.java | 5 +-
.../PriorityUtilizationQueueOrderingPolicy.java | 11 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 15 +
.../yarn/server/resourcemanager/MockNM.java | 8 +
.../yarn/server/resourcemanager/MockRM.java | 6 +
...alCapacityPreemptionPolicyMockFramework.java | 13 +
...estProportionalCapacityPreemptionPolicy.java | 29 +-
...pacityPreemptionPolicyIntraQueueWithDRF.java | 6 +-
.../TestAbsoluteResourceConfiguration.java | 516 +++++++++++++++++++
.../capacity/TestApplicationLimits.java | 30 +-
.../TestApplicationLimitsByPartition.java | 4 +
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/capacity/TestChildQueueOrder.java | 2 +
.../scheduler/capacity/TestLeafQueue.java | 261 ++++------
.../scheduler/capacity/TestParentQueue.java | 8 +
.../scheduler/capacity/TestReservations.java | 19 +
...tPriorityUtilizationQueueOrderingPolicy.java | 3 +
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
35 files changed, 1831 insertions(+), 434 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index cda5ec7..1be8a08 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1152,4 +1152,35 @@ public class StringUtils {
return s1.equalsIgnoreCase(s2);
}
+ /**
+ * <p>Checks if the String contains only unicode letters.</p>
+ *
+ * <p><code>null</code> will return <code>false</code>.
+ * An empty String (length()=0) will return <code>true</code>.</p>
+ *
+ * <pre>
+ * StringUtils.isAlpha(null) = false
+ * StringUtils.isAlpha("") = true
+ * StringUtils.isAlpha(" ") = false
+ * StringUtils.isAlpha("abc") = true
+ * StringUtils.isAlpha("ab2c") = false
+ * StringUtils.isAlpha("ab-c") = false
+ * </pre>
+ *
+ * @param str the String to check, may be null
+ * @return <code>true</code> if only contains letters, and is non-null
+ */
+ public static boolean isAlpha(String str) {
+ if (str == null) {
+ return false;
+ }
+ int sz = str.length();
+ for (int i = 0; i < sz; i++) {
+ if (Character.isLetter(str.charAt(i)) == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 7f155e7..aefa85c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -131,4 +131,10 @@ public class DefaultResourceCalculator extends ResourceCalculator {
public boolean isAnyMajorResourceZero(Resource resource) {
return resource.getMemorySize() == 0f;
}
+
+ @Override
+ public Resource normalizeDown(Resource r, Resource stepFactor) {
+ return Resources.createResource(
+ roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index ca828a5..dfda85c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -559,4 +559,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
public boolean isAnyMajorResourceZero(Resource resource) {
return resource.getMemorySize() == 0f || resource.getVirtualCores() == 0;
}
+
+ @Override
+ public Resource normalizeDown(Resource r, Resource stepFactor) {
+ return Resources.createResource(
+ roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
+ roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index d59560f..a816290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -235,4 +235,16 @@ public abstract class ResourceCalculator {
* @return returns true if any resource is zero.
*/
public abstract boolean isAnyMajorResourceZero(Resource resource);
+
+ /**
+ * Get resource <code>r</code>and normalize down using step-factor
+ * <code>stepFactor</code>.
+ *
+ * @param r
+ * resource to be multiplied
+ * @param stepFactor
+ * factor by which to normalize down
+ * @return resulting normalized resource
+ */
+ public abstract Resource normalizeDown(Resource r, Resource stepFactor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 793aebf..70c2e73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -494,4 +494,9 @@ public class Resources {
Resource resource) {
return rc.isAnyMajorResourceZero(resource);
}
+
+ public static Resource normalizeDown(ResourceCalculator calculator,
+ Resource resource, Resource factor) {
+ return calculator.normalizeDown(resource, factor);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index f843db4..748548a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -140,10 +140,10 @@ public class FifoCandidatesSelector
// Can try preempting AMContainers (still saving atmost
// maxAMCapacityForThisQueue AMResource's) if more resources are
// required to be preemptionCandidates from this Queue.
- Resource maxAMCapacityForThisQueue = Resources.multiply(
- Resources.multiply(clusterResource,
- leafQueue.getAbsoluteCapacity()),
- leafQueue.getMaxAMResourcePerQueuePercent());
+ Resource maxAMCapacityForThisQueue = Resources
+ .multiply(
+ leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL),
+ leafQueue.getMaxAMResourcePerQueuePercent());
preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist,
resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
@@ -199,7 +199,6 @@ public class FifoCandidatesSelector
* Given a target preemption for a specific application, select containers
* to preempt (after unreserving all reservation for that app).
*/
- @SuppressWarnings("unchecked")
private void preemptFrom(FiCaSchedulerApp app,
Resource clusterResource, Map<String, Resource> resToObtainByPartition,
List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index c4c98e2..72330c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolic
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -486,6 +487,13 @@ public class ProportionalCapacityPreemptionPolicy
float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt);
boolean preemptionDisabled = curQueue.getPreemptionDisabled();
+ QueueResourceQuotas queueResourceQuotas = curQueue
+ .getQueueResourceQuotas();
+ Resource effMinRes = queueResourceQuotas
+ .getEffectiveMinResource(partitionToLookAt);
+ Resource effMaxRes = queueResourceQuotas
+ .getEffectiveMaxResource(partitionToLookAt);
+
Resource current = Resources
.clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt));
Resource killable = Resources.none();
@@ -511,7 +519,7 @@ public class ProportionalCapacityPreemptionPolicy
ret = new TempQueuePerPartition(queueName, current, preemptionDisabled,
partitionToLookAt, killable, absCap, absMaxCap, partitionResource,
- reserved, curQueue);
+ reserved, curQueue, effMinRes, effMaxRes);
if (curQueue instanceof ParentQueue) {
String configuredOrderingPolicy =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 89452f9..bd236fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -48,6 +48,9 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
double normalizedGuarantee;
+ private Resource effMinRes;
+ private Resource effMaxRes;
+
final ArrayList<TempQueuePerPartition> children;
private Collection<TempAppPerPartition> apps;
LeafQueue leafQueue;
@@ -68,7 +71,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
TempQueuePerPartition(String queueName, Resource current,
boolean preemptionDisabled, String partition, Resource killable,
float absCapacity, float absMaxCapacity, Resource totalPartitionResource,
- Resource reserved, CSQueue queue) {
+ Resource reserved, CSQueue queue, Resource effMinRes,
+ Resource effMaxRes) {
super(queueName, current, Resource.newInstance(0, 0), reserved,
Resource.newInstance(0, 0));
@@ -95,6 +99,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
this.absCapacity = absCapacity;
this.absMaxCapacity = absMaxCapacity;
this.totalPartitionResource = totalPartitionResource;
+ this.effMinRes = effMinRes;
+ this.effMaxRes = effMaxRes;
}
public void setLeafQueue(LeafQueue l) {
@@ -177,10 +183,18 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
}
public Resource getGuaranteed() {
+ if(!effMinRes.equals(Resources.none())) {
+ return Resources.clone(effMinRes);
+ }
+
return Resources.multiply(totalPartitionResource, absCapacity);
}
public Resource getMax() {
+ if(!effMaxRes.equals(Resources.none())) {
+ return Resources.clone(effMaxRes);
+ }
+
return Resources.multiply(totalPartitionResource, absMaxCapacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
new file mode 100644
index 0000000..c295323
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class AbstractResourceUsage {
+ protected ReadLock readLock;
+ protected WriteLock writeLock;
+ protected Map<String, UsageByLabel> usages;
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public AbstractResourceUsage() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ usages = new HashMap<String, UsageByLabel>();
+ usages.put(NL, new UsageByLabel(NL));
+ }
+
+ // Usage enum here to make implement cleaner
+ public enum ResourceType {
+ // CACHED_USED and CACHED_PENDING may be read by anyone, but must only
+ // be written by ordering policies
+ USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING(
+ 5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE(
+ 9), EFF_MAX_RESOURCE(
+ 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12);
+
+ private int idx;
+
+ private ResourceType(int value) {
+ this.idx = value;
+ }
+ }
+
+ public static class UsageByLabel {
+ // usage by label, contains all UsageType
+ private Resource[] resArr;
+
+ public UsageByLabel(String label) {
+ resArr = new Resource[ResourceType.values().length];
+ for (int i = 0; i < resArr.length; i++) {
+ resArr[i] = Resource.newInstance(0, 0);
+ };
+ }
+
+ public Resource getUsed() {
+ return resArr[ResourceType.USED.idx];
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{used=" + resArr[0] + "%, ");
+ sb.append("pending=" + resArr[1] + "%, ");
+ sb.append("am_used=" + resArr[2] + "%, ");
+ sb.append("reserved=" + resArr[3] + "%}");
+ sb.append("min_eff=" + resArr[9] + "%, ");
+ sb.append("max_eff=" + resArr[10] + "%}");
+ sb.append("min_effup=" + resArr[11] + "%, ");
+ return sb.toString();
+ }
+ }
+
+ private static Resource normalize(Resource res) {
+ if (res == null) {
+ return Resources.none();
+ }
+ return res;
+ }
+
+ protected Resource _get(String label, ResourceType type) {
+ if (label == null) {
+ label = RMNodeLabelsManager.NO_LABEL;
+ }
+
+ try {
+ readLock.lock();
+ UsageByLabel usage = usages.get(label);
+ if (null == usage) {
+ return Resources.none();
+ }
+ return normalize(usage.resArr[type.idx]);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ protected Resource _getAll(ResourceType type) {
+ try {
+ readLock.lock();
+ Resource allOfType = Resources.createResource(0);
+ for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
+ //all usages types are initialized
+ Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
+ }
+ return allOfType;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private UsageByLabel getAndAddIfMissing(String label) {
+ if (label == null) {
+ label = RMNodeLabelsManager.NO_LABEL;
+ }
+ if (!usages.containsKey(label)) {
+ UsageByLabel u = new UsageByLabel(label);
+ usages.put(label, u);
+ return u;
+ }
+
+ return usages.get(label);
+ }
+
+ protected void _set(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.resArr[type.idx] = res;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected void _inc(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ Resources.addTo(usage.resArr[type.idx], res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected void _dec(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ Resources.subtractFrom(usage.resArr[type.idx], res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ try {
+ readLock.lock();
+ return usages.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public Set<String> getNodePartitionsSet() {
+ try {
+ readLock.lock();
+ return usages.keySet();
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
new file mode 100644
index 0000000..2e653fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+/**
+ * QueueResourceQuotas by Labels for following fields by label
+ * - EFFECTIVE_MIN_CAPACITY
+ * - EFFECTIVE_MAX_CAPACITY
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class QueueResourceQuotas extends AbstractResourceUsage {
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public QueueResourceQuotas() {
+ super();
+ }
+
+ /*
+ * Configured Minimum Resource
+ */
+ public Resource getConfiguredMinResource() {
+ return _get(NL, ResourceType.MIN_RESOURCE);
+ }
+
+ public Resource getConfiguredMinResource(String label) {
+ return _get(label, ResourceType.MIN_RESOURCE);
+ }
+
+ public void setConfiguredMinResource(String label, Resource res) {
+ _set(label, ResourceType.MIN_RESOURCE, res);
+ }
+
+ public void setConfiguredMinResource(Resource res) {
+ _set(NL, ResourceType.MIN_RESOURCE, res);
+ }
+
+ /*
+ * Configured Maximum Resource
+ */
+ public Resource getConfiguredMaxResource() {
+ return getConfiguredMaxResource(NL);
+ }
+
+ public Resource getConfiguredMaxResource(String label) {
+ return _get(label, ResourceType.MAX_RESOURCE);
+ }
+
+ public void setConfiguredMaxResource(Resource res) {
+ setConfiguredMaxResource(NL, res);
+ }
+
+ public void setConfiguredMaxResource(String label, Resource res) {
+ _set(label, ResourceType.MAX_RESOURCE, res);
+ }
+
+ /*
+ * Effective Minimum Resource
+ */
+ public Resource getEffectiveMinResource() {
+ return _get(NL, ResourceType.EFF_MIN_RESOURCE);
+ }
+
+ public Resource getEffectiveMinResource(String label) {
+ return _get(label, ResourceType.EFF_MIN_RESOURCE);
+ }
+
+ public void setEffectiveMinResource(String label, Resource res) {
+ _set(label, ResourceType.EFF_MIN_RESOURCE, res);
+ }
+
+ public void setEffectiveMinResource(Resource res) {
+ _set(NL, ResourceType.EFF_MIN_RESOURCE, res);
+ }
+
+ /*
+ * Effective Maximum Resource
+ */
+ public Resource getEffectiveMaxResource() {
+ return getEffectiveMaxResource(NL);
+ }
+
+ public Resource getEffectiveMaxResource(String label) {
+ return _get(label, ResourceType.EFF_MAX_RESOURCE);
+ }
+
+ public void setEffectiveMaxResource(Resource res) {
+ setEffectiveMaxResource(NL, res);
+ }
+
+ public void setEffectiveMaxResource(String label, Resource res) {
+ _set(label, ResourceType.EFF_MAX_RESOURCE, res);
+ }
+
+ /*
+ * Effective Minimum Resource
+ */
+ public Resource getEffectiveMinResourceUp() {
+ return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
+ }
+
+ public Resource getEffectiveMinResourceUp(String label) {
+ return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
+ }
+
+ public void setEffectiveMinResourceUp(String label, Resource res) {
+ _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
+ }
+
+ public void setEffectiveMinResourceUp(Resource res) {
+ _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
+ }
+
+ /*
+ * Effective Maximum Resource
+ */
+ public Resource getEffectiveMaxResourceUp() {
+ return getEffectiveMaxResourceUp(NL);
+ }
+
+ public Resource getEffectiveMaxResourceUp(String label) {
+ return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
+ }
+
+ public void setEffectiveMaxResourceUp(Resource res) {
+ setEffectiveMaxResourceUp(NL, res);
+ }
+
+ public void setEffectiveMaxResourceUp(String label, Resource res) {
+ _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 6f0c7d2..ede4aec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -39,63 +39,12 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*
* And it is thread-safe
*/
-public class ResourceUsage {
- private ReadLock readLock;
- private WriteLock writeLock;
- private Map<String, UsageByLabel> usages;
+public class ResourceUsage extends AbstractResourceUsage {
// short for no-label :)
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
- private final UsageByLabel usageNoLabel;
public ResourceUsage() {
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- readLock = lock.readLock();
- writeLock = lock.writeLock();
-
- usages = new HashMap<String, UsageByLabel>();
- usageNoLabel = new UsageByLabel(NL);
- usages.put(NL, usageNoLabel);
- }
-
- // Usage enum here to make implement cleaner
- private enum ResourceType {
- //CACHED_USED and CACHED_PENDING may be read by anyone, but must only
- //be written by ordering policies
- USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
- CACHED_PENDING(5), AMLIMIT(6);
-
- private int idx;
-
- private ResourceType(int value) {
- this.idx = value;
- }
- }
-
- private static class UsageByLabel {
- // usage by label, contains all UsageType
- private Resource[] resArr;
-
- public UsageByLabel(String label) {
- resArr = new Resource[ResourceType.values().length];
- for (int i = 0; i < resArr.length; i++) {
- resArr[i] = Resource.newInstance(0, 0);
- };
- }
-
- public Resource getUsed() {
- return resArr[ResourceType.USED.idx];
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{used=" + resArr[0] + "%, ");
- sb.append("pending=" + resArr[1] + "%, ");
- sb.append("am_used=" + resArr[2] + "%, ");
- sb.append("reserved=" + resArr[3] + "%}");
- sb.append("am_limit=" + resArr[6] + "%, ");
- return sb.toString();
- }
+ super();
}
/*
@@ -109,22 +58,6 @@ public class ResourceUsage {
return _get(label, ResourceType.USED);
}
- public Resource getCachedUsed() {
- return _get(NL, ResourceType.CACHED_USED);
- }
-
- public Resource getCachedUsed(String label) {
- return _get(label, ResourceType.CACHED_USED);
- }
-
- public Resource getCachedPending() {
- return _get(NL, ResourceType.CACHED_PENDING);
- }
-
- public Resource getCachedPending(String label) {
- return _get(label, ResourceType.CACHED_PENDING);
- }
-
public void incUsed(String label, Resource res) {
_inc(label, ResourceType.USED, res);
}
@@ -145,7 +78,7 @@ public class ResourceUsage {
setUsed(NL, res);
}
- public void copyAllUsed(ResourceUsage other) {
+ public void copyAllUsed(AbstractResourceUsage other) {
try {
writeLock.lock();
for (Entry<String, UsageByLabel> entry : other.usages.entrySet()) {
@@ -160,22 +93,6 @@ public class ResourceUsage {
_set(label, ResourceType.USED, res);
}
- public void setCachedUsed(String label, Resource res) {
- _set(label, ResourceType.CACHED_USED, res);
- }
-
- public void setCachedUsed(Resource res) {
- _set(NL, ResourceType.CACHED_USED, res);
- }
-
- public void setCachedPending(String label, Resource res) {
- _set(label, ResourceType.CACHED_PENDING, res);
- }
-
- public void setCachedPending(Resource res) {
- _set(NL, ResourceType.CACHED_PENDING, res);
- }
-
/*
* Pending
*/
@@ -281,6 +198,47 @@ public class ResourceUsage {
_set(label, ResourceType.AMUSED, res);
}
+ public Resource getAllPending() {
+ return _getAll(ResourceType.PENDING);
+ }
+
+ public Resource getAllUsed() {
+ return _getAll(ResourceType.USED);
+ }
+
+ // Cache Used
+ public Resource getCachedUsed() {
+ return _get(NL, ResourceType.CACHED_USED);
+ }
+
+ public Resource getCachedUsed(String label) {
+ return _get(label, ResourceType.CACHED_USED);
+ }
+
+ public Resource getCachedPending() {
+ return _get(NL, ResourceType.CACHED_PENDING);
+ }
+
+ public Resource getCachedPending(String label) {
+ return _get(label, ResourceType.CACHED_PENDING);
+ }
+
+ public void setCachedUsed(String label, Resource res) {
+ _set(label, ResourceType.CACHED_USED, res);
+ }
+
+ public void setCachedUsed(Resource res) {
+ _set(NL, ResourceType.CACHED_USED, res);
+ }
+
+ public void setCachedPending(String label, Resource res) {
+ _set(label, ResourceType.CACHED_PENDING, res);
+ }
+
+ public void setCachedPending(Resource res) {
+ _set(NL, ResourceType.CACHED_PENDING, res);
+ }
+
/*
* AM-Resource Limit
*/
@@ -316,94 +274,6 @@ public class ResourceUsage {
_set(label, ResourceType.AMLIMIT, res);
}
- private static Resource normalize(Resource res) {
- if (res == null) {
- return Resources.none();
- }
- return res;
- }
-
- private Resource _get(String label, ResourceType type) {
- if (label == null || label.equals(NL)) {
- return normalize(usageNoLabel.resArr[type.idx]);
- }
- try {
- readLock.lock();
- UsageByLabel usage = usages.get(label);
- if (null == usage) {
- return Resources.none();
- }
- return normalize(usage.resArr[type.idx]);
- } finally {
- readLock.unlock();
- }
- }
-
- private Resource _getAll(ResourceType type) {
- try {
- readLock.lock();
- Resource allOfType = Resources.createResource(0);
- for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
- //all usages types are initialized
- Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
- }
- return allOfType;
- } finally {
- readLock.unlock();
- }
- }
-
- public Resource getAllPending() {
- return _getAll(ResourceType.PENDING);
- }
-
- public Resource getAllUsed() {
- return _getAll(ResourceType.USED);
- }
-
- private UsageByLabel getAndAddIfMissing(String label) {
- if (label == null || label.equals(NL)) {
- return usageNoLabel;
- }
- if (!usages.containsKey(label)) {
- UsageByLabel u = new UsageByLabel(label);
- usages.put(label, u);
- return u;
- }
-
- return usages.get(label);
- }
-
- private void _set(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- usage.resArr[type.idx] = res;
- } finally {
- writeLock.unlock();
- }
- }
-
- private void _inc(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- Resources.addTo(usage.resArr[type.idx], res);
- } finally {
- writeLock.unlock();
- }
- }
-
- private void _dec(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- Resources.subtractFrom(usage.resArr[type.idx], res);
- } finally {
- writeLock.unlock();
- }
- }
-
public Resource getCachedDemand(String label) {
try {
readLock.lock();
@@ -415,23 +285,4 @@ public class ResourceUsage {
readLock.unlock();
}
}
-
- @Override
- public String toString() {
- try {
- readLock.lock();
- return usages.toString();
- } finally {
- readLock.unlock();
- }
- }
-
- public Set<String> getNodePartitionsSet() {
- try {
- readLock.lock();
- return usages.keySet();
- } finally {
- readLock.unlock();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 250f4e6..47ec620 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@@ -87,6 +89,7 @@ public abstract class AbstractCSQueue implements CSQueue {
final ResourceCalculator resourceCalculator;
Set<String> accessibleLabels;
+ Set<String> resourceTypes;
final RMNodeLabelsManager labelManager;
String defaultLabelExpression;
@@ -102,6 +105,14 @@ public abstract class AbstractCSQueue implements CSQueue {
// etc.
QueueCapacities queueCapacities;
+ QueueResourceQuotas queueResourceQuotas;
+
+ protected enum CapacityConfigType {
+ NONE, PERCENTAGE, ABSOLUTE_RESOURCE
+ };
+ protected CapacityConfigType capacityConfigType =
+ CapacityConfigType.NONE;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
@@ -141,6 +152,9 @@ public abstract class AbstractCSQueue implements CSQueue {
// initialize QueueCapacities
queueCapacities = new QueueCapacities(parent == null);
+ // initialize queueResourceQuotas
+ queueResourceQuotas = new QueueResourceQuotas();
+
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
@@ -276,6 +290,10 @@ public abstract class AbstractCSQueue implements CSQueue {
this.defaultLabelExpression =
csContext.getConfiguration().getDefaultNodeLabelExpression(
getQueuePath());
+ this.resourceTypes = new HashSet<String>();
+ for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
+ resourceTypes.add(type.toString().toLowerCase());
+ }
// inherit from parent if labels not set
if (this.accessibleLabels == null && parent != null) {
@@ -292,6 +310,11 @@ public abstract class AbstractCSQueue implements CSQueue {
// After we setup labels, we can setup capacities
setupConfigurableCapacities();
+ // Also fetch minimum/maximum resource constraint for this queue if
+ // configured.
+ capacityConfigType = CapacityConfigType.NONE;
+ updateConfigurableResourceRequirement(getQueuePath(), clusterResource);
+
this.maximumAllocation =
csContext.getConfiguration().getMaximumAllocationPerQueue(
getQueuePath());
@@ -364,6 +387,125 @@ public abstract class AbstractCSQueue implements CSQueue {
return unionInheritedWeights;
}
+ protected void updateConfigurableResourceRequirement(String queuePath,
+ Resource clusterResource) {
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ Set<String> configuredNodelabels = conf.getConfiguredNodeLabels(queuePath);
+
+ for (String label : configuredNodelabels) {
+ Resource minResource = conf.getMinimumResourceRequirement(label,
+ queuePath, resourceTypes);
+ Resource maxResource = conf.getMaximumResourceRequirement(label,
+ queuePath, resourceTypes);
+
+ if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
+ this.capacityConfigType = (!minResource.equals(Resources.none())
+ && queueCapacities.getAbsoluteCapacity(label) == 0f)
+ ? CapacityConfigType.ABSOLUTE_RESOURCE
+ : CapacityConfigType.PERCENTAGE;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("capacityConfigType is updated as '" + capacityConfigType
+ + "' for queue '" + getQueueName());
+ }
+ }
+
+ validateAbsoluteVsPercentageCapacityConfig(minResource);
+
+ // If min resource for a resource type is greater than its max resource,
+ // throw exception to handle such error configs.
+ if (!maxResource.equals(Resources.none()) && Resources.greaterThan(
+ resourceCalculator, clusterResource, minResource, maxResource)) {
+ throw new IllegalArgumentException("Min resource configuration "
+ + minResource + " is greater than its max value:" + maxResource
+ + " in queue:" + getQueueName());
+ }
+
+ // If parent's max resource is lesser to a specific child's max
+ // resource, throw exception to handle such error configs.
+ if (parent != null) {
+ Resource parentMaxRes = parent.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ parentMaxRes, Resources.none())) {
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ maxResource, parentMaxRes)) {
+ throw new IllegalArgumentException("Max resource configuration "
+ + maxResource + " is greater than parents max value:"
+ + parentMaxRes + " in queue:" + getQueueName());
+ }
+
+ // If child's max resource is not set, but its parent max resource is
+ // set, we must set child max resource to its parent's.
+ if (maxResource.equals(Resources.none())
+ && !minResource.equals(Resources.none())) {
+ maxResource = Resources.clone(parentMaxRes);
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating absolute resource configuration for queue:"
+ + getQueueName() + " as minResource=" + minResource
+ + " and maxResource=" + maxResource);
+ }
+
+ queueResourceQuotas.setConfiguredMinResource(label, minResource);
+ queueResourceQuotas.setConfiguredMaxResource(label, maxResource);
+ }
+ }
+
+ private void validateAbsoluteVsPercentageCapacityConfig(
+ Resource minResource) {
+ CapacityConfigType localType = CapacityConfigType.PERCENTAGE;
+ if (!minResource.equals(Resources.none())) {
+ localType = CapacityConfigType.ABSOLUTE_RESOURCE;
+ }
+
+ if (!queueName.equals("root")
+ && !this.capacityConfigType.equals(localType)) {
+ throw new IllegalArgumentException("Queue '" + getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource.");
+ }
+ }
+
+ @Override
+ public CapacityConfigType getCapacityConfigType() {
+ return capacityConfigType;
+ }
+
+ @Override
+ public Resource getEffectiveCapacity(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMinResource(label));
+ }
+
+ @Override
+ public Resource getEffectiveCapacityUp(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
+ }
+
+ @Override
+ public Resource getEffectiveCapacityDown(String label, Resource factor) {
+ return Resources.normalizeDown(resourceCalculator,
+ getQueueResourceQuotas().getEffectiveMinResource(label),
+ minimumAllocation);
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacity(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMaxResource(label));
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacityDown(String label, Resource factor) {
+ return Resources.normalizeDown(resourceCalculator,
+ getQueueResourceQuotas().getEffectiveMaxResource(label),
+ minimumAllocation);
+ }
+
private void initializeQueueState(QueueState previousState,
QueueState configuredState, QueueState parentState) {
// verify that we can not any value for State other than RUNNING/STOPPED
@@ -555,6 +697,11 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
+ public QueueResourceQuotas getQueueResourceQuotas() {
+ return queueResourceQuotas;
+ }
+
+ @Override
public ReentrantReadWriteLock.ReadLock getReadLock() {
return readLock;
}
@@ -604,7 +751,7 @@ public abstract class AbstractCSQueue implements CSQueue {
* limit-set-by-parent)
*/
Resource queueMaxResource =
- getQueueMaxResource(nodePartition, clusterResource);
+ getQueueMaxResource(nodePartition);
return Resources.min(resourceCalculator, clusterResource,
queueMaxResource, currentResourceLimits.getLimit());
@@ -617,11 +764,8 @@ public abstract class AbstractCSQueue implements CSQueue {
return Resources.none();
}
- Resource getQueueMaxResource(String nodePartition, Resource clusterResource) {
- return Resources.multiplyAndNormalizeDown(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- queueCapacities.getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource getQueueMaxResource(String nodePartition) {
+ return getEffectiveMaxCapacity(nodePartition);
}
public boolean hasChildQueues() {
@@ -782,7 +926,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.incUsed(nodeLabel, resourceToInc);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- nodeLabel, this);
+ Resources.none(), nodeLabel, this);
if (null != parent) {
parent.incUsedResource(nodeLabel, resourceToInc, null);
}
@@ -798,7 +942,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.decUsed(nodeLabel, resourceToDec);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- nodeLabel, this);
+ Resources.none(), nodeLabel, this);
if (null != parent) {
parent.decUsedResource(nodeLabel, resourceToDec, null);
}
@@ -904,7 +1048,7 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maxResourceLimit;
if (allocation.getSchedulingMode()
== SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
- maxResourceLimit = getQueueMaxResource(partition, cluster);
+ maxResourceLimit = getQueueMaxResource(partition);
} else{
maxResourceLimit = labelManager.getResourceByLabel(
schedulerContainer.getNodePartition(), cluster);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 3a17d1b..a93d74e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -37,21 +37,20 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.security.PrivilegedEntity;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
/**
* <code>CSQueue</code> represents a node in the tree of
@@ -357,4 +356,41 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* @return map of usernames and corresponding weight
*/
Map<String, Float> getUserWeights();
+
+ /**
+ * Get QueueResourceQuotas associated with each queue.
+ * @return QueueResourceQuotas
+ */
+ public QueueResourceQuotas getQueueResourceQuotas();
+
+ /**
+ * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE
+ * @return CapacityConfigType
+ */
+ public CapacityConfigType getCapacityConfigType();
+
+ /**
+ * Get effective capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @return effective queue capacity
+ */
+ Resource getEffectiveCapacity(String label);
+ Resource getEffectiveCapacityUp(String label);
+ Resource getEffectiveCapacityDown(String label, Resource factor);
+
+ /**
+ * Get effective max capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @return effective max queue capacity
+ */
+ Resource getEffectiveMaxCapacity(String label);
+ Resource getEffectiveMaxCapacityDown(String label, Resource factor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index e1014c1..81dec80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -150,7 +150,7 @@ class CSQueueUtils {
}
}
}
-
+
// Set absolute capacities for {capacity, maximum-capacity}
private static void updateAbsoluteCapacitiesByNodeLabels(
QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
@@ -180,8 +180,8 @@ class CSQueueUtils {
* used resource for all partitions of this queue.
*/
public static void updateUsedCapacity(final ResourceCalculator rc,
- final Resource totalPartitionResource, String nodePartition,
- AbstractCSQueue childQueue) {
+ final Resource totalPartitionResource, Resource clusterResource,
+ String nodePartition, AbstractCSQueue childQueue) {
QueueCapacities queueCapacities = childQueue.getQueueCapacities();
CSQueueMetrics queueMetrics = childQueue.getMetrics();
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
@@ -193,11 +193,8 @@ class CSQueueUtils {
if (Resources.greaterThan(rc, totalPartitionResource,
totalPartitionResource, Resources.none())) {
- // queueGuaranteed = totalPartitionedResource *
- // absolute_capacity(partition)
- Resource queueGuranteedResource =
- Resources.multiply(totalPartitionResource,
- queueCapacities.getAbsoluteCapacity(nodePartition));
+ Resource queueGuranteedResource = childQueue
+ .getEffectiveCapacity(nodePartition);
// make queueGuranteed >= minimum_allocation to avoid divided by 0.
queueGuranteedResource =
@@ -248,9 +245,7 @@ class CSQueueUtils {
for (String partition : nodeLabels) {
// Calculate guaranteed resource for a label in a queue by below logic.
// (total label resource) * (absolute capacity of label in that queue)
- Resource queueGuranteedResource = Resources.multiply(nlm
- .getResourceByLabel(partition, cluster), queue.getQueueCapacities()
- .getAbsoluteCapacity(partition));
+ Resource queueGuranteedResource = queue.getEffectiveCapacity(partition);
// Available resource in queue for a specific label will be calculated as
// {(guaranteed resource for a label in a queue) -
@@ -289,15 +284,14 @@ class CSQueueUtils {
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
if (nodePartition == null) {
- for (String partition : Sets.union(
- queueCapacities.getNodePartitionsSet(),
+ for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
queueResourceUsage.getNodePartitionsSet())) {
updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
- partition, childQueue);
+ cluster, partition, childQueue);
}
} else {
updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
- nodePartition, childQueue);
+ cluster, nodePartition, childQueue);
}
// Update queue metrics w.r.t node labels. In a generic way, we can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3a519ec..bfbc418 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -60,6 +61,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.Set;
import java.util.StringTokenizer;
@@ -316,6 +319,21 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
+ /** Configuring absolute min/max resources in a queue **/
+ @Private
+ public static final String MINIMUM_RESOURCE = "min-resource";
+
+ @Private
+ public static final String MAXIMUM_RESOURCE = "max-resource";
+
+ public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores";
+
+ public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)";
+
+ public enum AbsoluteResourceType {
+ MEMORY, VCORES;
+ }
+
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
public CapacitySchedulerConfiguration() {
@@ -393,7 +411,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public float getNonLabeledQueueCapacity(String queue) {
float capacity = queue.equals("root") ? 100.0f : getFloat(
- getQueuePrefix(queue) + CAPACITY, UNDEFINED);
+ getQueuePrefix(queue) + CAPACITY, 0f);
if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
throw new IllegalArgumentException("Illegal " +
"capacity of " + capacity + " for queue " + queue);
@@ -1522,4 +1540,163 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
setLong(getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, defaultLifetime);
}
+
+ public static String getUnits(String resourceValue) {
+ String units;
+ for (int i = 0; i < resourceValue.length(); i++) {
+ if (Character.isAlphabetic(resourceValue.charAt(i))) {
+ units = resourceValue.substring(i);
+ if (StringUtils.isAlpha(units)) {
+ return units;
+ }
+ }
+ }
+ return "";
+ }
+
+ /**
+ * Get absolute minimum resource requirement for a queue.
+ *
+ * @param label
+ * NodeLabel
+ * @param queue
+ * queue path
+ * @param resourceTypes
+ * Resource types
+ * @return ResourceInformation
+ */
+ public Resource getMinimumResourceRequirement(String label, String queue,
+ Set<String> resourceTypes) {
+ return internalGetLabeledResourceRequirementForQueue(queue, label,
+ resourceTypes, MINIMUM_RESOURCE);
+ }
+
+ /**
+ * Get absolute maximum resource requirement for a queue.
+ *
+ * @param label
+ * NodeLabel
+ * @param queue
+ * queue path
+ * @param resourceTypes
+ * Resource types
+ * @return Resource
+ */
+ public Resource getMaximumResourceRequirement(String label, String queue,
+ Set<String> resourceTypes) {
+ return internalGetLabeledResourceRequirementForQueue(queue, label,
+ resourceTypes, MAXIMUM_RESOURCE);
+ }
+
+ @VisibleForTesting
+ public void setMinimumResourceRequirement(String label, String queue,
+ Resource resource) {
+ updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE);
+ }
+
+ @VisibleForTesting
+ public void setMaximumResourceRequirement(String label, String queue,
+ Resource resource) {
+ updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE);
+ }
+
+ private void updateMinMaxResourceToConf(String label, String queue,
+ Resource resource, String type) {
+ if (queue.equals("root")) {
+ throw new IllegalArgumentException(
+ "Cannot set resource, root queue will take 100% of cluster capacity");
+ }
+
+ StringBuilder resourceString = new StringBuilder();
+ resourceString
+ .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
+ + resource.getMemorySize() + ","
+ + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
+ + resource.getVirtualCores() + "]");
+
+ String prefix = getQueuePrefix(queue) + type;
+ if (!label.isEmpty()) {
+ prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label
+ + DOT + type;
+ }
+ set(prefix, resourceString.toString());
+ }
+
+ private Resource internalGetLabeledResourceRequirementForQueue(String queue,
+ String label, Set<String> resourceTypes, String suffix) {
+ String propertyName = getNodeLabelPrefix(queue, label) + suffix;
+ String resourceString = get(propertyName);
+ if (resourceString == null || resourceString.isEmpty()) {
+ return Resources.none();
+ }
+
+ // Define resource here.
+ Resource resource = Resource.newInstance(0l, 0);
+ Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE)
+ .matcher(resourceString);
+ /*
+ * Absolute resource configuration for a queue will be grouped by "[]".
+ * Syntax of absolute resource config could be like below
+ * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores".
+ */
+ if (matcher.find()) {
+ // Get the sub-group.
+ String subGroup = matcher.group(1);
+ if (subGroup.trim().isEmpty()) {
+ return Resources.none();
+ }
+
+ for (String kvPair : subGroup.trim().split(",")) {
+ String[] splits = kvPair.split("=");
+
+ // Ensure that each sub string is key value pair separated by '='.
+ if (splits != null && splits.length > 1) {
+ updateResourceValuesFromConfig(resourceTypes, resource, splits);
+ }
+ }
+ }
+
+ // Memory has to be configured always.
+ if (resource.getMemorySize() == 0l) {
+ return Resources.none();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CSConf - getAbsolueResourcePerQueue: prefix="
+ + getNodeLabelPrefix(queue, label) + ", capacity=" + resource);
+ }
+ return resource;
+ }
+
+ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
+ Resource resource, String[] splits) {
+
+ // If key is not a valid type, skip it.
+ if (!resourceTypes.contains(splits[0])) {
+ return;
+ }
+
+ String units = getUnits(splits[1]);
+ Long resourceValue = Long
+ .valueOf(splits[1].substring(0, splits[1].length() - units.length()));
+
+ // Convert all incoming units to MB if units is configured.
+ if (!units.isEmpty()) {
+ resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
+ }
+
+ // map it based on key.
+ AbsoluteResourceType resType = AbsoluteResourceType
+ .valueOf(StringUtils.toUpperCase(splits[0].trim()));
+ switch (resType) {
+ case MEMORY :
+ resource.setMemorySize(resourceValue);
+ break;
+ case VCORES :
+ resource.setVirtualCores(resourceValue.intValue());
+ break;
+ default :
+ break;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f24e30a..b58d04a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,12 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
- Resource queuePartitionResource = Resources
- .multiplyAndNormalizeUp(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition,
- lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition),
- minimumAllocation);
+ Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
Resource userAMLimit = Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
@@ -705,11 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
* non-labeled), * with per-partition am-resource-percent to get the max am
* resource limit for this queue and partition.
*/
- Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition),
- minimumAllocation);
+ Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
Resource queueCurrentLimit = Resources.none();
// For non-labeled partition, we need to consider the current queue
@@ -965,6 +956,14 @@ public class LeafQueue extends AbstractCSQueue {
private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
// Set preemption-allowed:
// For leaf queue, only under-utilized queue is allowed to preempt resources from other queues
+ if (!queueResourceQuotas.getEffectiveMinResource(nodePartition)
+ .equals(Resources.none())) {
+ limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator,
+ csContext.getClusterResource(), queueUsage.getUsed(nodePartition),
+ queueResourceQuotas.getEffectiveMinResource(nodePartition)));
+ return;
+ }
+
float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition);
float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition);
limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
@@ -1341,7 +1340,7 @@ public class LeafQueue extends AbstractCSQueue {
currentPartitionResourceLimit =
partition.equals(RMNodeLabelsManager.NO_LABEL)
? currentPartitionResourceLimit
- : getQueueMaxResource(partition, clusterResource);
+ : getQueueMaxResource(partition);
Resource headroom = Resources.componentwiseMin(
Resources.subtract(userLimitResource, user.getUsed(partition)),
@@ -1713,12 +1712,8 @@ public class LeafQueue extends AbstractCSQueue {
// this. So need cap limits by queue's max capacity here.
this.cachedResourceLimitsForHeadroom =
new ResourceLimits(currentResourceLimits.getLimit());
- Resource queueMaxResource =
- Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
- .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
- queueCapacities
- .getAbsoluteMaximumCapacity(RMNodeLabelsManager.NO_LABEL),
- minimumAllocation);
+ Resource queueMaxResource = getEffectiveMaxCapacityDown(
+ RMNodeLabelsManager.NO_LABEL, minimumAllocation);
this.cachedResourceLimitsForHeadroom.setLimit(Resources.min(
resourceCalculator, clusterResource, queueMaxResource,
currentResourceLimits.getLimit()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 6800b74..257f834 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -162,31 +164,78 @@ public class ParentQueue extends AbstractCSQueue {
writeLock.lock();
// Validate
float childCapacities = 0;
+ Resource minResDefaultLabel = Resources.createResource(0, 0);
for (CSQueue queue : childQueues) {
childCapacities += queue.getCapacity();
+ Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource());
+
+ // If any child queue is using percentage based capacity model vs parent
+ // queues' absolute configuration or vice versa, throw back an
+ // exception.
+ if (!queueName.equals("root") && getCapacity() != 0f
+ && !queue.getQueueResourceQuotas().getConfiguredMinResource()
+ .equals(Resources.none())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource together.");
+ }
}
+
float delta = Math.abs(1.0f - childCapacities); // crude way to check
// allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || (
- (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
- throw new IllegalArgumentException(
- "Illegal" + " capacity of " + childCapacities
- + " for children of queue " + queueName);
+ if ((minResDefaultLabel.equals(Resources.none())
+ && (queueCapacities.getCapacity() > 0) && (delta > PRECISION))
+ || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
+ throw new IllegalArgumentException("Illegal" + " capacity of "
+ + childCapacities + " for children of queue " + queueName);
}
// check label capacities
for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
// check children's labels
float sum = 0;
+ Resource minRes = Resources.createResource(0, 0);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+ scheduler.getClusterResource());
for (CSQueue queue : childQueues) {
sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+
+ // If any child queue of a label is using percentage based capacity
+ // model vs parent queues' absolute configuration or vice versa, throw
+ // back an exception
+ if (!queueName.equals("root") && !this.capacityConfigType
+ .equals(queue.getCapacityConfigType())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + "configuration or absolute resource together for label:"
+ + nodeLabel);
+ }
+
+ // Accumulate all min/max resource configured for all child queues.
+ Resources.addTo(minRes, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource(nodeLabel));
}
- if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
+ if ((minResDefaultLabel.equals(Resources.none()) && capacityByLabel > 0
+ && Math.abs(1.0f - sum) > PRECISION)
|| (capacityByLabel == 0) && (sum > 0)) {
throw new IllegalArgumentException(
"Illegal" + " capacity of " + sum + " for children of queue "
+ queueName + " for label=" + nodeLabel);
}
+
+ // Ensure that for each parent queue: parent.min-resource >=
+ // Σ(child.min-resource).
+ Resource parentMinResource = queueResourceQuotas
+ .getConfiguredMinResource(nodeLabel);
+ if (!parentMinResource.equals(Resources.none()) && Resources.lessThan(
+ resourceCalculator, resourceByLabel, parentMinResource, minRes)) {
+ throw new IllegalArgumentException("Parent Queues" + " capacity: "
+ + parentMinResource + " is less than" + " to its children:"
+ + minRes + " for queue:" + queueName);
+ }
}
this.childQueues.clear();
@@ -686,11 +735,8 @@ public class ParentQueue extends AbstractCSQueue {
child.getQueueResourceUsage().getUsed(nodePartition));
// Get child's max resource
- Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
+ nodePartition, minimumAllocation);
// Child's limit should be capped by child configured max resource
childLimit =
@@ -826,6 +872,14 @@ public class ParentQueue extends AbstractCSQueue {
ResourceLimits resourceLimits) {
try {
writeLock.lock();
+
+ // Update effective capacity in all parent queue.
+ Set<String> configuredNodelabels = csContext.getConfiguration()
+ .getConfiguredNodeLabels(getQueuePath());
+ for (String label : configuredNodelabels) {
+ calculateEffectiveResourcesAndCapacity(label, clusterResource);
+ }
+
// Update all children
for (CSQueue childQueue : childQueues) {
// Get ResourceLimits of child queue before assign containers
@@ -847,6 +901,110 @@ public class ParentQueue extends AbstractCSQueue {
return true;
}
+ private void calculateEffectiveResourcesAndCapacity(String label,
+ Resource clusterResource) {
+
+ // For root queue, ensure that max/min resource is updated to latest
+ // cluster resource.
+ Resource resourceByLabel = labelManager.getResourceByLabel(label,
+ clusterResource);
+ if (getQueueName().equals("root")) {
+ queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel);
+ queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ }
+
+ // Total configured min resources of direct children of queue
+ Resource configuredMinResources = Resource.newInstance(0L, 0);
+ for (CSQueue childQueue : getChildQueues()) {
+ Resources.addTo(configuredMinResources,
+ childQueue.getQueueResourceQuotas().getConfiguredMinResource(label));
+ }
+
+ // Factor to scale down effective resource: When cluster has sufficient
+ // resources, effective_min_resources will be same as configured
+ // min_resources.
+ float effectiveMinRatio = 1;
+ ResourceCalculator rc = this.csContext.getResourceCalculator();
+ if (getQueueName().equals("root")) {
+ if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
+ clusterResource, resourceByLabel, configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ resourceByLabel, configuredMinResources);
+ }
+ } else {
+ if (Resources.lessThan(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources);
+ }
+ }
+
+ // loop and do this for all child queues
+ for (CSQueue childQueue : getChildQueues()) {
+ Resource minResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMinResource(label);
+
+ // Update effective resource (min/max) to each child queue.
+ if (childQueue.getCapacityConfigType()
+ .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(minResource, effectiveMinRatio));
+
+ // Max resource of a queue should be a minimum of {configuredMaxRes,
+ // parentMaxRes}. parentMaxRes could be configured value. But if not
+ // present could also be taken from effective max resource of parent.
+ Resource parentMaxRes = queueResourceQuotas
+ .getConfiguredMaxResource(label);
+ if (parentMaxRes.equals(Resources.none())) {
+ parentMaxRes = parent.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label);
+ }
+
+ // Minimum of {childMaxResource, parentMaxRes}. However if
+ // childMaxResource is empty, consider parent's max resource alone.
+ Resource childMaxResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ Resource effMaxResource = Resources.min(resourceCalculator,
+ resourceByLabel, childMaxResource.equals(Resources.none())
+ ? parentMaxRes
+ : childMaxResource,
+ parentMaxRes);
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.clone(effMaxResource));
+ } else {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label)));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label)));
+
+ childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label),
+ minimumAllocation));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc,
+ resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label),
+ minimumAllocation));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating effective min resource for queue:"
+ + childQueue.getQueueName() + " as effMinResource="
+ + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label)
+ + "and Updating effective max resource as effMaxResource="
+ + childQueue.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label));
+ }
+ }
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
@@ -979,9 +1137,21 @@ public class ParentQueue extends AbstractCSQueue {
* When this happens, we have to preempt killable container (on same or different
* nodes) of parent queue to avoid violating parent's max resource.
*/
- if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
- < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
- killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+ if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition)
+ .equals(Resources.none())) {
+ if (Resources.lessThan(resourceCalculator, clusterResource,
+ queueResourceQuotas.getEffectiveMaxResource(nodePartition),
+ queueUsage.getUsed(nodePartition))) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
+ } else {
+ if (getQueueCapacities()
+ .getAbsoluteMaximumCapacity(nodePartition) < getQueueCapacities()
+ .getAbsoluteUsedCapacity(nodePartition)) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
}
} finally {
writeLock.unlock();
@@ -998,8 +1168,7 @@ public class ParentQueue extends AbstractCSQueue {
Resource partitionResource = labelManager.getResourceByLabel(partition,
null);
- Resource maxResource = Resources.multiply(partitionResource,
- getQueueCapacities().getAbsoluteMaximumCapacity(partition));
+ Resource maxResource = getEffectiveMaxCapacity(partition);
while (Resources.greaterThan(resourceCalculator, partitionResource,
queueUsage.getUsed(partition), maxResource)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc00b7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 33f30b0..efc20e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,10 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = Resources.multiplyAndNormalizeUp(
- resourceCalculator, partitionResource,
- lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition),
- lQueue.getMinimumAllocation());
+ Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-7296.
convertToProtoFormat(Resource r) is not setting for all resource types.
Contributed by lovekesh bansal.
Posted by su...@apache.org.
YARN-7296. convertToProtoFormat(Resource r) is not setting for all resource types. Contributed by lovekesh bansal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41351b05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41351b05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41351b05
Branch: refs/heads/YARN-5881
Commit: 41351b05f32fe0c69dd7e1a4c4c18922fc857f4f
Parents: 5cfbad6
Author: Sunil G <su...@apache.org>
Authored: Tue Oct 10 12:02:18 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Oct 10 12:02:18 2017 +0530
----------------------------------------------------------------------
.../impl/pb/GetAllResourceProfilesResponsePBImpl.java | 6 ++----
.../impl/pb/GetResourceProfileResponsePBImpl.java | 9 +++------
.../yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java | 7 ++-----
3 files changed, 7 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41351b05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java
index eaa392f..6eb0dff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceProfilesResponsePBImpl.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProfilesProto;
@@ -129,9 +130,6 @@ public class GetAllResourceProfilesResponsePBImpl
}
private ResourceProto convertToProtoFormat(Resource res) {
- ResourcePBImpl r = new ResourcePBImpl();
- r.setMemorySize(res.getMemorySize());
- r.setVirtualCores(res.getVirtualCores());
- return r.getProto();
+ return ProtoUtils.convertToProtoFormat(res);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41351b05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java
index e08d077..d30de3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetResourceProfileResponsePBImpl.java
@@ -22,10 +22,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProtoOrBuilder;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -92,10 +92,7 @@ public class GetResourceProfileResponsePBImpl
}
private ResourceProto convertToProtoFormat(Resource res) {
- ResourcePBImpl r = new ResourcePBImpl();
- r.setMemorySize(res.getMemorySize());
- r.setVirtualCores(res.getVirtualCores());
- return r.getProto();
+ return ProtoUtils.convertToProtoFormat(res);
}
private void maybeInitBuilder() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41351b05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java
index 8c161f8..cfe1e8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProfileCapabilityPBImpl.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos;
-import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProtoOrBuilder;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -121,9 +121,6 @@ public class ProfileCapabilityPBImpl extends ProfileCapability {
}
private YarnProtos.ResourceProto convertToProtoFormat(Resource res) {
- ResourcePBImpl r = new ResourcePBImpl();
- r.setMemorySize(res.getMemorySize());
- r.setVirtualCores(res.getVirtualCores());
- return r.getProto();
+ return ProtoUtils.convertToProtoFormat(res);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: Merge branch 'YARN-5734' into
trunk. (xgong)
Posted by su...@apache.org.
Merge branch 'YARN-5734' into trunk. (xgong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9ff17f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9ff17f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9ff17f5
Branch: refs/heads/YARN-5881
Commit: f9ff17f510517ef6c6acc1e1179a68e771df1847
Parents: 7938208 1d36b53
Author: Xuan <xg...@apache.org>
Authored: Mon Oct 9 14:46:02 2017 -0700
Committer: Xuan <xg...@apache.org>
Committed: Mon Oct 9 14:46:02 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 +
.../hadoop/yarn/conf/YarnConfiguration.java | 61 +++
.../hadoop/yarn/client/cli/SchedConfCLI.java | 254 ++++++++++
.../yarn/client/cli/TestSchedConfCLI.java | 158 ++++++
.../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 57 +++
.../yarn/webapp/dao/SchedConfUpdateInfo.java | 85 ++++
.../hadoop/yarn/webapp/dao/package-info.java | 27 +
.../yarn/webapp/util/YarnWebServiceUtils.java | 14 +
.../src/main/resources/yarn-default.xml | 64 +++
.../server/resourcemanager/AdminService.java | 24 +-
.../server/resourcemanager/ResourceManager.java | 24 +-
.../RMStateVersionIncompatibleException.java | 2 +-
.../recovery/ZKRMStateStore.java | 5 +-
.../ConfigurationMutationACLPolicy.java | 47 ++
.../ConfigurationMutationACLPolicyFactory.java | 49 ++
.../DefaultConfigurationMutationACLPolicy.java | 45 ++
.../scheduler/MutableConfScheduler.java | 53 ++
.../scheduler/MutableConfigurationProvider.java | 66 +++
.../scheduler/capacity/CapacityScheduler.java | 66 ++-
.../capacity/CapacitySchedulerContext.java | 6 +
.../capacity/CapacitySchedulerQueueManager.java | 11 +-
.../capacity/conf/CSConfigurationProvider.java | 47 ++
.../conf/FileBasedCSConfigurationProvider.java | 67 +++
.../conf/InMemoryConfigurationStore.java | 95 ++++
.../conf/LeveldbConfigurationStore.java | 322 ++++++++++++
.../conf/MutableCSConfigurationProvider.java | 284 +++++++++++
...ueueAdminConfigurationMutationACLPolicy.java | 110 ++++
.../capacity/conf/YarnConfigurationStore.java | 178 +++++++
.../conf/YarnConfigurationStoreFactory.java | 46 ++
.../capacity/conf/ZKConfigurationStore.java | 235 +++++++++
.../scheduler/capacity/conf/package-info.java | 29 ++
.../resourcemanager/webapp/RMWebServices.java | 66 ++-
.../resourcemanager/TestRMAdminService.java | 23 +
.../TestConfigurationMutationACLPolicies.java | 172 +++++++
.../capacity/TestCapacityScheduler.java | 4 +-
.../conf/ConfigurationStoreBaseTest.java | 92 ++++
.../conf/TestInMemoryConfigurationStore.java | 30 ++
.../conf/TestLeveldbConfigurationStore.java | 211 ++++++++
.../TestMutableCSConfigurationProvider.java | 102 ++++
.../capacity/conf/TestZKConfigurationStore.java | 412 +++++++++++++++
.../TestRMWebServicesConfigurationMutation.java | 507 +++++++++++++++++++
.../src/site/markdown/CapacityScheduler.md | 25 +-
.../src/site/markdown/ResourceManagerRest.md | 188 +++++++
.../src/site/markdown/YarnCommands.md | 13 +
45 files changed, 4338 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-13102. Update GroupsMapping
documentation to reflect the new changes. Contributed by Esther Kundin.
Posted by su...@apache.org.
HADOOP-13102. Update GroupsMapping documentation to reflect the new changes. Contributed by Esther Kundin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/075358eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/075358eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/075358eb
Branch: refs/heads/YARN-5881
Commit: 075358eb6fff5ae4a40ac4dfde292e2a9a4ceddf
Parents: 8bcc49e
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Oct 11 15:58:20 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Oct 11 15:58:20 2017 -0700
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/GroupsMapping.md | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/075358eb/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index 89aca16..806ed54 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -85,9 +85,10 @@ This file should be readable only by the Unix user running the daemons.
It is possible to set a maximum time limit when searching and awaiting a result.
Set `hadoop.security.group.mapping.ldap.directory.search.timeout` to 0 if infinite wait period is desired. Default is 10,000 milliseconds (10 seconds).
+This is the limit for each ldap query. If `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` is set to a positive value, then the total latency will be bounded by max(Recur Depth in LDAP, `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` ) * `hadoop.security.group.mapping.ldap.directory.search.timeout`.
-The implementation does not attempt to resolve group hierarchies. Therefore, a user must be an explicit member of a group object
-in order to be considered a member.
+`hadoop.security.group.mapping.ldap.base` configures how far to walk up the groups hierarchy when resolving groups.
+By default, with a limit of 0, in order to be considered a member of a group, the user must be an explicit member in LDAP. Otherwise, it will traverse the group hierarchy `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` levels up.
### Active Directory ###
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-7046. Add closing logic to
configuration store
Posted by su...@apache.org.
YARN-7046. Add closing logic to configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6622daa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6622daa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6622daa
Branch: refs/heads/YARN-5881
Commit: d6622daaa3c95a5040d06f91dc0c793bafa70f95
Parents: ff39c0d
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Fri Sep 22 10:18:27 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../scheduler/MutableConfigurationProvider.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 3 +
.../conf/LeveldbConfigurationStore.java | 44 ++--
.../conf/MutableCSConfigurationProvider.java | 6 +
.../capacity/conf/YarnConfigurationStore.java | 6 +
.../conf/ConfigurationStoreBaseTest.java | 2 +
.../conf/TestLeveldbConfigurationStore.java | 213 +++++++++++++++++++
7 files changed, 270 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index f8e8814..2b9b25a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import java.io.IOException;
+
/**
* Interface for allowing changing scheduler configurations.
*/
@@ -55,4 +57,10 @@ public interface MutableConfigurationProvider {
* @throws Exception if confirming mutation fails
*/
void confirmPendingMutation(boolean isValid) throws Exception;
+
+ /**
+ * Closes the configuration provider, releasing any required resources.
+ * @throws IOException on failure to close
+ */
+ void close() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 16b27c1..de95179 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -411,6 +411,9 @@ public class CapacityScheduler extends
writeLock.unlock();
}
+ if (isConfigurationMutable()) {
+ ((MutableConfigurationProvider) csConfProvider).close();
+ }
super.serviceStop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 1b0eb9f..21de7a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -72,7 +73,8 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
private long maxLogs;
private Configuration conf;
private LogMutation pendingMutation;
- private static final Version CURRENT_VERSION_INFO = Version
+ @VisibleForTesting
+ protected static final Version CURRENT_VERSION_INFO = Version
.newInstance(0, 1);
private Timer compactionTimer;
private long compactionIntervalMsec;
@@ -82,7 +84,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
RMContext rmContext) throws IOException {
this.conf = config;
try {
- this.db = initDatabase(schedConf);
+ initDatabase(schedConf);
this.maxLogs = config.getLong(
YarnConfiguration.RM_SCHEDCONF_MAX_LOGS,
YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS);
@@ -96,7 +98,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
}
}
- private DB initDatabase(Configuration config) throws Exception {
+ private void initDatabase(Configuration config) throws Exception {
Path storeRoot = createStorageDir();
Options options = new Options();
options.createIfMissing(false);
@@ -108,13 +110,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
if (key1Str.equals(key2Str)) {
return 0;
} else if (key1Str.equals(VERSION_KEY)) {
- return -1;
- } else if (key2Str.equals(VERSION_KEY)) {
return 1;
- } else if (key1Str.equals(LOG_KEY)) {
+ } else if (key2Str.equals(VERSION_KEY)) {
return -1;
- } else if (key2Str.equals(LOG_KEY)) {
+ } else if (key1Str.equals(LOG_KEY)) {
return 1;
+ } else if (key2Str.equals(LOG_KEY)) {
+ return -1;
}
return key1Str.compareTo(key2Str);
}
@@ -156,7 +158,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
throw e;
}
}
- return db;
}
private Path createStorageDir() throws IOException {
@@ -176,6 +177,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
}
@Override
+ public void close() throws IOException {
+ if (db != null) {
+ db.close();
+ }
+ }
+
+ @Override
public void logMutation(LogMutation logMutation) throws IOException {
LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
logs.add(logMutation);
@@ -212,8 +220,12 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return baos.toByteArray();
}
}
+
private LinkedList<LogMutation> deserLogMutations(byte[] mutations) throws
IOException {
+ if (mutations == null) {
+ return new LinkedList<>();
+ }
try (ObjectInput input = new ObjectInputStream(
new ByteArrayInputStream(mutations))) {
return (LinkedList<LogMutation>) input.readObject();
@@ -225,13 +237,16 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
@Override
public synchronized Configuration retrieve() {
DBIterator itr = db.iterator();
- itr.seek(bytes(LOG_KEY));
+ itr.seekToFirst();
Configuration config = new Configuration(false);
- itr.next();
while (itr.hasNext()) {
Map.Entry<byte[], byte[]> entry = itr.next();
- config.set(new String(entry.getKey(), StandardCharsets.UTF_8),
- new String(entry.getValue(), StandardCharsets.UTF_8));
+ String key = new String(entry.getKey(), StandardCharsets.UTF_8);
+ String value = new String(entry.getValue(), StandardCharsets.UTF_8);
+ if (key.equals(LOG_KEY) || key.equals(VERSION_KEY)) {
+ break;
+ }
+ config.set(key, value);
}
return config;
}
@@ -268,6 +283,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return version;
}
+ @VisibleForTesting
+ protected LinkedList<LogMutation> getLogs() throws Exception {
+ return deserLogMutations(db.get(bytes(LOG_KEY)));
+ }
+
@Override
public void storeVersion() throws Exception {
String key = VERSION_KEY;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 70d1840..ccadf76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -92,6 +92,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
try {
confStore.initialize(config, schedConf, rmContext);
+ confStore.checkVersion();
} catch (Exception e) {
throw new IOException(e);
}
@@ -103,6 +104,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
aclMutationPolicy.init(config, rmContext);
}
+ @Override
+ public void close() throws IOException {
+ confStore.close();
+ }
+
@VisibleForTesting
public YarnConfigurationStore getConfStore() {
return confStore;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 1356535..7fb52fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -96,6 +96,12 @@ public abstract class YarnConfigurationStore {
RMContext rmContext) throws Exception;
/**
+ * Closes the configuration store, releasing any required resources.
+ * @throws IOException on failure to close
+ */
+ public void close() throws IOException {}
+
+ /**
* Logs the configuration change to backing store.
* @param logMutation configuration change to be persisted in write ahead log
* @throws IOException if logging fails
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
index bbe9570..8f3bc71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
@@ -71,6 +71,7 @@ public abstract class ConfigurationStoreBaseTest {
confStore.confirmMutation(false);
assertNull("Configuration should not be updated",
confStore.retrieve().get("keyUpdate2"));
+ confStore.close();
}
@Test
@@ -86,5 +87,6 @@ public abstract class ConfigurationStoreBaseTest {
confStore.logMutation(mutation);
confStore.confirmMutation(true);
assertNull(confStore.retrieve().get("key"));
+ confStore.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6622daa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
new file mode 100644
index 0000000..779208a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Tests {@link LeveldbConfigurationStore}.
+ */
+public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestLeveldbConfigurationStore.class);
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")),
+ TestLeveldbConfigurationStore.class.getName());
+
+ private ResourceManager rm;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ FileUtil.fullyDelete(TEST_DIR);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.LEVELDB_CONFIGURATION_STORE);
+ conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString());
+ }
+
+ @Test
+ public void testVersioning() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ assertEquals(LeveldbConfigurationStore.CURRENT_VERSION_INFO,
+ confStore.getConfStoreVersion());
+ confStore.close();
+ }
+
+ @Test
+ public void testPersistConfiguration() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+
+ // Create a new configuration store, and check for old configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+ @Test
+ public void testPersistUpdatedConfiguration() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.retrieve().get("key"));
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key", "val");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+ confStore.confirmMutation(true);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+
+ // Create a new configuration store, and check for updated configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+ @Test
+ public void testMaxLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
+ confStore.initialize(conf, schedConf, rmContext);
+ LinkedList<YarnConfigurationStore.LogMutation> logs =
+ ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(0, logs.size());
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("key1", "val1");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update1, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ confStore.confirmMutation(true);
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("key2", "val2");
+ mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+
+ // Next update should purge first update from logs.
+ Map<String, String> update3 = new HashMap<>();
+ update3.put("key3", "val3");
+ mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.close();
+ }
+
+ /**
+ * When restarting, RM should read from current state of store, including
+ * any updates from the previous RM instance.
+ * @throws Exception
+ */
+ @Test
+ public void testRestartReadsFromUpdatedStore() throws Exception {
+ ResourceManager rm1 = new MockRM(conf);
+ rm1.start();
+ assertNull(((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+
+ // Update configuration on RM
+ SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo();
+ schedConfUpdateInfo.getGlobalParams().put("key", "val");
+ MutableConfigurationProvider confProvider = ((MutableConfScheduler)
+ rm1.getResourceScheduler()).getMutableConfProvider();
+ UserGroupInformation user = UserGroupInformation
+ .createUserForTesting(TEST_USER, new String[0]);
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf, rm1.getRMContext());
+ assertEquals("val", ((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+ confProvider.confirmPendingMutation(true);
+ assertEquals("val", ((MutableCSConfigurationProvider) confProvider)
+ .getConfStore().retrieve().get("key"));
+ // Next update is not persisted, it should not be recovered
+ schedConfUpdateInfo.getGlobalParams().put("key", "badVal");
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.close();
+
+ // Start RM2 and verifies it starts with updated configuration
+ ResourceManager rm2 = new MockRM(conf);
+ rm2.start();
+ assertEquals("val", ((MutableCSConfigurationProvider) (
+ (CapacityScheduler) rm2.getResourceScheduler())
+ .getMutableConfProvider()).getConfStore().retrieve().get("key"));
+ assertEquals("val", ((MutableConfScheduler) rm2.getResourceScheduler())
+ .getConfiguration().get("key"));
+ rm2.close();
+ }
+
+ @Override
+ public YarnConfigurationStore createConfStore() {
+ return new LeveldbConfigurationStore();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-6620. Add support in
NodeManager to isolate GPU devices by using CGroups. Contributed by Wangda
Tan.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
new file mode 100644
index 0000000..e22597d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+public class TestGpuDeviceInformationParser {
+ @Test
+ public void testParse() throws IOException, YarnException {
+ File f = new File("src/test/resources/nvidia-smi-sample-xml-output");
+ String s = FileUtils.readFileToString(f, "UTF-8");
+
+ GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+
+ GpuDeviceInformation info = parser.parseXml(s);
+ Assert.assertEquals("375.66", info.getDriverVersion());
+ Assert.assertEquals(2, info.getGpus().size());
+ PerGpuDeviceInformation gpu1 = info.getGpus().get(1);
+ Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName());
+ Assert.assertEquals(16384, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
+ Assert.assertEquals(10.3f,
+ gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6);
+ Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6);
+ Assert.assertEquals(85f, gpu1.getTemperature().getMaxGpuTemp(), 1e-6);
+ Assert.assertEquals(82f, gpu1.getTemperature().getSlowThresholdGpuTemp(),
+ 1e-6);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output
new file mode 100644
index 0000000..5ccb722
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/nvidia-smi-sample-xml-output
@@ -0,0 +1,547 @@
+<?xml version="1.0" ?>
+<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v8.dtd">
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<nvidia_smi_log>
+ <timestamp>Wed Sep 6 21:52:51 2017</timestamp>
+ <driver_version>375.66</driver_version>
+ <attached_gpus>2</attached_gpus>
+ <gpu id="0000:04:00.0">
+ <product_name>Tesla P100-PCIE-12GB</product_name>
+ <product_brand>Tesla</product_brand>
+ <display_mode>Disabled</display_mode>
+ <display_active>Disabled</display_active>
+ <persistence_mode>Disabled</persistence_mode>
+ <accounting_mode>Disabled</accounting_mode>
+ <accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
+ <driver_model>
+ <current_dm>N/A</current_dm>
+ <pending_dm>N/A</pending_dm>
+ </driver_model>
+ <serial>0320717030197</serial>
+ <uuid>GPU-28604e81-21ec-cc48-6759-bf2648b22e16</uuid>
+ <minor_number>0</minor_number>
+ <vbios_version>86.00.3A.00.02</vbios_version>
+ <multigpu_board>No</multigpu_board>
+ <board_id>0x400</board_id>
+ <gpu_part_number>900-2H400-0110-030</gpu_part_number>
+ <inforom_version>
+ <img_version>H400.0202.00.01</img_version>
+ <oem_object>1.1</oem_object>
+ <ecc_object>4.1</ecc_object>
+ <pwr_object>N/A</pwr_object>
+ </inforom_version>
+ <gpu_operation_mode>
+ <current_gom>N/A</current_gom>
+ <pending_gom>N/A</pending_gom>
+ </gpu_operation_mode>
+ <gpu_virtualization_mode>
+ <virtualization_mode>None</virtualization_mode>
+ </gpu_virtualization_mode>
+ <pci>
+ <pci_bus>04</pci_bus>
+ <pci_device>00</pci_device>
+ <pci_domain>0000</pci_domain>
+ <pci_device_id>15F710DE</pci_device_id>
+ <pci_bus_id>0000:04:00.0</pci_bus_id>
+ <pci_sub_system_id>11DA10DE</pci_sub_system_id>
+ <pci_gpu_link_info>
+ <pcie_gen>
+ <max_link_gen>3</max_link_gen>
+ <current_link_gen>3</current_link_gen>
+ </pcie_gen>
+ <link_widths>
+ <max_link_width>16x</max_link_width>
+ <current_link_width>16x</current_link_width>
+ </link_widths>
+ </pci_gpu_link_info>
+ <pci_bridge_chip>
+ <bridge_chip_type>N/A</bridge_chip_type>
+ <bridge_chip_fw>N/A</bridge_chip_fw>
+ </pci_bridge_chip>
+ <replay_counter>0</replay_counter>
+ <tx_util>0 KB/s</tx_util>
+ <rx_util>0 KB/s</rx_util>
+ </pci>
+ <fan_speed>N/A</fan_speed>
+ <performance_state>P0</performance_state>
+ <clocks_throttle_reasons>
+ <clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
+ <clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
+ <clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
+ <clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
+ <clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
+ <clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
+ </clocks_throttle_reasons>
+ <fb_memory_usage>
+ <total>12193 MiB</total>
+ <used>0 MiB</used>
+ <free>12193 MiB</free>
+ </fb_memory_usage>
+ <bar1_memory_usage>
+ <total>16384 MiB</total>
+ <used>2 MiB</used>
+ <free>16382 MiB</free>
+ </bar1_memory_usage>
+ <compute_mode>Default</compute_mode>
+ <utilization>
+ <gpu_util>0 %</gpu_util>
+ <memory_util>0 %</memory_util>
+ <encoder_util>0 %</encoder_util>
+ <decoder_util>0 %</decoder_util>
+ </utilization>
+ <encoder_stats>
+ <session_count>0</session_count>
+ <average_fps>0</average_fps>
+ <average_latency>0 ms</average_latency>
+ </encoder_stats>
+ <ecc_mode>
+ <current_ecc>Enabled</current_ecc>
+ <pending_ecc>Enabled</pending_ecc>
+ </ecc_mode>
+ <ecc_errors>
+ <volatile>
+ <single_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </single_bit>
+ <double_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </double_bit>
+ </volatile>
+ <aggregate>
+ <single_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </single_bit>
+ <double_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </double_bit>
+ </aggregate>
+ </ecc_errors>
+ <retired_pages>
+ <multiple_single_bit_retirement>
+ <retired_count>0</retired_count>
+ <retired_page_addresses>
+ </retired_page_addresses>
+ </multiple_single_bit_retirement>
+ <double_bit_retirement>
+ <retired_count>0</retired_count>
+ <retired_page_addresses>
+ </retired_page_addresses>
+ </double_bit_retirement>
+ <pending_retirement>No</pending_retirement>
+ </retired_pages>
+ <temperature>
+ <gpu_temp>31 C</gpu_temp>
+ <gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
+ <gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
+ </temperature>
+ <power_readings>
+ <power_state>P0</power_state>
+ <power_management>Supported</power_management>
+ <power_draw>24.84 W</power_draw>
+ <power_limit>250.00 W</power_limit>
+ <default_power_limit>250.00 W</default_power_limit>
+ <enforced_power_limit>250.00 W</enforced_power_limit>
+ <min_power_limit>125.00 W</min_power_limit>
+ <max_power_limit>250.00 W</max_power_limit>
+ </power_readings>
+ <clocks>
+ <graphics_clock>405 MHz</graphics_clock>
+ <sm_clock>405 MHz</sm_clock>
+ <mem_clock>715 MHz</mem_clock>
+ <video_clock>835 MHz</video_clock>
+ </clocks>
+ <applications_clocks>
+ <graphics_clock>1189 MHz</graphics_clock>
+ <mem_clock>715 MHz</mem_clock>
+ </applications_clocks>
+ <default_applications_clocks>
+ <graphics_clock>1189 MHz</graphics_clock>
+ <mem_clock>715 MHz</mem_clock>
+ </default_applications_clocks>
+ <max_clocks>
+ <graphics_clock>1328 MHz</graphics_clock>
+ <sm_clock>1328 MHz</sm_clock>
+ <mem_clock>715 MHz</mem_clock>
+ <video_clock>1328 MHz</video_clock>
+ </max_clocks>
+ <clock_policy>
+ <auto_boost>N/A</auto_boost>
+ <auto_boost_default>N/A</auto_boost_default>
+ </clock_policy>
+ <supported_clocks>
+ <supported_mem_clock>
+ <value>715 MHz</value>
+ <supported_graphics_clock>1328 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1316 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1303 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1290 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1278 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1265 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1252 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1240 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1227 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1215 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1202 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1189 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1177 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1164 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1151 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1139 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1126 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1113 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1101 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1088 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1075 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1063 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1050 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1037 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1025 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1012 MHz</supported_graphics_clock>
+ <supported_graphics_clock>999 MHz</supported_graphics_clock>
+ <supported_graphics_clock>987 MHz</supported_graphics_clock>
+ <supported_graphics_clock>974 MHz</supported_graphics_clock>
+ <supported_graphics_clock>961 MHz</supported_graphics_clock>
+ <supported_graphics_clock>949 MHz</supported_graphics_clock>
+ <supported_graphics_clock>936 MHz</supported_graphics_clock>
+ <supported_graphics_clock>923 MHz</supported_graphics_clock>
+ <supported_graphics_clock>911 MHz</supported_graphics_clock>
+ <supported_graphics_clock>898 MHz</supported_graphics_clock>
+ <supported_graphics_clock>885 MHz</supported_graphics_clock>
+ <supported_graphics_clock>873 MHz</supported_graphics_clock>
+ <supported_graphics_clock>860 MHz</supported_graphics_clock>
+ <supported_graphics_clock>847 MHz</supported_graphics_clock>
+ <supported_graphics_clock>835 MHz</supported_graphics_clock>
+ <supported_graphics_clock>822 MHz</supported_graphics_clock>
+ <supported_graphics_clock>810 MHz</supported_graphics_clock>
+ <supported_graphics_clock>797 MHz</supported_graphics_clock>
+ <supported_graphics_clock>784 MHz</supported_graphics_clock>
+ <supported_graphics_clock>772 MHz</supported_graphics_clock>
+ <supported_graphics_clock>759 MHz</supported_graphics_clock>
+ <supported_graphics_clock>746 MHz</supported_graphics_clock>
+ <supported_graphics_clock>734 MHz</supported_graphics_clock>
+ <supported_graphics_clock>721 MHz</supported_graphics_clock>
+ <supported_graphics_clock>708 MHz</supported_graphics_clock>
+ <supported_graphics_clock>696 MHz</supported_graphics_clock>
+ <supported_graphics_clock>683 MHz</supported_graphics_clock>
+ <supported_graphics_clock>670 MHz</supported_graphics_clock>
+ <supported_graphics_clock>658 MHz</supported_graphics_clock>
+ <supported_graphics_clock>645 MHz</supported_graphics_clock>
+ <supported_graphics_clock>632 MHz</supported_graphics_clock>
+ <supported_graphics_clock>620 MHz</supported_graphics_clock>
+ <supported_graphics_clock>607 MHz</supported_graphics_clock>
+ <supported_graphics_clock>594 MHz</supported_graphics_clock>
+ <supported_graphics_clock>582 MHz</supported_graphics_clock>
+ <supported_graphics_clock>569 MHz</supported_graphics_clock>
+ <supported_graphics_clock>556 MHz</supported_graphics_clock>
+ <supported_graphics_clock>544 MHz</supported_graphics_clock>
+ </supported_mem_clock>
+ </supported_clocks>
+ <processes>
+ </processes>
+ <accounted_processes>
+ </accounted_processes>
+ </gpu>
+
+ <gpu id="0000:82:00.0">
+ <product_name>Tesla P100-PCIE-12GB</product_name>
+ <product_brand>Tesla</product_brand>
+ <display_mode>Disabled</display_mode>
+ <display_active>Disabled</display_active>
+ <persistence_mode>Disabled</persistence_mode>
+ <accounting_mode>Disabled</accounting_mode>
+ <accounting_mode_buffer_size>1920</accounting_mode_buffer_size>
+ <driver_model>
+ <current_dm>N/A</current_dm>
+ <pending_dm>N/A</pending_dm>
+ </driver_model>
+ <serial>0320717031755</serial>
+ <uuid>GPU-46915a82-3fd2-8e11-ae26-a80b607c04f3</uuid>
+ <minor_number>1</minor_number>
+ <vbios_version>86.00.3A.00.02</vbios_version>
+ <multigpu_board>No</multigpu_board>
+ <board_id>0x8200</board_id>
+ <gpu_part_number>900-2H400-0110-030</gpu_part_number>
+ <inforom_version>
+ <img_version>H400.0202.00.01</img_version>
+ <oem_object>1.1</oem_object>
+ <ecc_object>4.1</ecc_object>
+ <pwr_object>N/A</pwr_object>
+ </inforom_version>
+ <gpu_operation_mode>
+ <current_gom>N/A</current_gom>
+ <pending_gom>N/A</pending_gom>
+ </gpu_operation_mode>
+ <gpu_virtualization_mode>
+ <virtualization_mode>None</virtualization_mode>
+ </gpu_virtualization_mode>
+ <pci>
+ <pci_bus>82</pci_bus>
+ <pci_device>00</pci_device>
+ <pci_domain>0000</pci_domain>
+ <pci_device_id>15F710DE</pci_device_id>
+ <pci_bus_id>0000:82:00.0</pci_bus_id>
+ <pci_sub_system_id>11DA10DE</pci_sub_system_id>
+ <pci_gpu_link_info>
+ <pcie_gen>
+ <max_link_gen>3</max_link_gen>
+ <current_link_gen>3</current_link_gen>
+ </pcie_gen>
+ <link_widths>
+ <max_link_width>16x</max_link_width>
+ <current_link_width>16x</current_link_width>
+ </link_widths>
+ </pci_gpu_link_info>
+ <pci_bridge_chip>
+ <bridge_chip_type>N/A</bridge_chip_type>
+ <bridge_chip_fw>N/A</bridge_chip_fw>
+ </pci_bridge_chip>
+ <replay_counter>0</replay_counter>
+ <tx_util>0 KB/s</tx_util>
+ <rx_util>0 KB/s</rx_util>
+ </pci>
+ <fan_speed>N/A</fan_speed>
+ <performance_state>P0</performance_state>
+ <clocks_throttle_reasons>
+ <clocks_throttle_reason_gpu_idle>Active</clocks_throttle_reason_gpu_idle>
+ <clocks_throttle_reason_applications_clocks_setting>Not Active</clocks_throttle_reason_applications_clocks_setting>
+ <clocks_throttle_reason_sw_power_cap>Not Active</clocks_throttle_reason_sw_power_cap>
+ <clocks_throttle_reason_hw_slowdown>Not Active</clocks_throttle_reason_hw_slowdown>
+ <clocks_throttle_reason_sync_boost>Not Active</clocks_throttle_reason_sync_boost>
+ <clocks_throttle_reason_unknown>Not Active</clocks_throttle_reason_unknown>
+ </clocks_throttle_reasons>
+ <fb_memory_usage>
+ <total>12193 MiB</total>
+ <used>0 MiB</used>
+ <free>12193 MiB</free>
+ </fb_memory_usage>
+ <bar1_memory_usage>
+ <total>16384 MiB</total>
+ <used>2 MiB</used>
+ <free>16382 MiB</free>
+ </bar1_memory_usage>
+ <compute_mode>Default</compute_mode>
+ <utilization>
+ <gpu_util>10.3 %</gpu_util>
+ <memory_util>0 %</memory_util>
+ <encoder_util>0 %</encoder_util>
+ <decoder_util>0 %</decoder_util>
+ </utilization>
+ <encoder_stats>
+ <session_count>0</session_count>
+ <average_fps>0</average_fps>
+ <average_latency>0 ms</average_latency>
+ </encoder_stats>
+ <ecc_mode>
+ <current_ecc>Enabled</current_ecc>
+ <pending_ecc>Enabled</pending_ecc>
+ </ecc_mode>
+ <ecc_errors>
+ <volatile>
+ <single_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </single_bit>
+ <double_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </double_bit>
+ </volatile>
+ <aggregate>
+ <single_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </single_bit>
+ <double_bit>
+ <device_memory>0</device_memory>
+ <register_file>0</register_file>
+ <l1_cache>N/A</l1_cache>
+ <l2_cache>0</l2_cache>
+ <texture_memory>0</texture_memory>
+ <texture_shm>0</texture_shm>
+ <total>0</total>
+ </double_bit>
+ </aggregate>
+ </ecc_errors>
+ <retired_pages>
+ <multiple_single_bit_retirement>
+ <retired_count>0</retired_count>
+ <retired_page_addresses>
+ </retired_page_addresses>
+ </multiple_single_bit_retirement>
+ <double_bit_retirement>
+ <retired_count>0</retired_count>
+ <retired_page_addresses>
+ </retired_page_addresses>
+ </double_bit_retirement>
+ <pending_retirement>No</pending_retirement>
+ </retired_pages>
+ <temperature>
+ <gpu_temp>34 C</gpu_temp>
+ <gpu_temp_max_threshold>85 C</gpu_temp_max_threshold>
+ <gpu_temp_slow_threshold>82 C</gpu_temp_slow_threshold>
+ </temperature>
+ <power_readings>
+ <power_state>P0</power_state>
+ <power_management>Supported</power_management>
+ <power_draw>25.54 W</power_draw>
+ <power_limit>250.00 W</power_limit>
+ <default_power_limit>250.00 W</default_power_limit>
+ <enforced_power_limit>250.00 W</enforced_power_limit>
+ <min_power_limit>125.00 W</min_power_limit>
+ <max_power_limit>250.00 W</max_power_limit>
+ </power_readings>
+ <clocks>
+ <graphics_clock>405 MHz</graphics_clock>
+ <sm_clock>405 MHz</sm_clock>
+ <mem_clock>715 MHz</mem_clock>
+ <video_clock>835 MHz</video_clock>
+ </clocks>
+ <applications_clocks>
+ <graphics_clock>1189 MHz</graphics_clock>
+ <mem_clock>715 MHz</mem_clock>
+ </applications_clocks>
+ <default_applications_clocks>
+ <graphics_clock>1189 MHz</graphics_clock>
+ <mem_clock>715 MHz</mem_clock>
+ </default_applications_clocks>
+ <max_clocks>
+ <graphics_clock>1328 MHz</graphics_clock>
+ <sm_clock>1328 MHz</sm_clock>
+ <mem_clock>715 MHz</mem_clock>
+ <video_clock>1328 MHz</video_clock>
+ </max_clocks>
+ <clock_policy>
+ <auto_boost>N/A</auto_boost>
+ <auto_boost_default>N/A</auto_boost_default>
+ </clock_policy>
+ <supported_clocks>
+ <supported_mem_clock>
+ <value>715 MHz</value>
+ <supported_graphics_clock>1328 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1316 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1303 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1290 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1278 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1265 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1252 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1240 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1227 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1215 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1202 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1189 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1177 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1164 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1151 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1139 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1126 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1113 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1101 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1088 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1075 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1063 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1050 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1037 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1025 MHz</supported_graphics_clock>
+ <supported_graphics_clock>1012 MHz</supported_graphics_clock>
+ <supported_graphics_clock>999 MHz</supported_graphics_clock>
+ <supported_graphics_clock>987 MHz</supported_graphics_clock>
+ <supported_graphics_clock>974 MHz</supported_graphics_clock>
+ <supported_graphics_clock>961 MHz</supported_graphics_clock>
+ <supported_graphics_clock>949 MHz</supported_graphics_clock>
+ <supported_graphics_clock>936 MHz</supported_graphics_clock>
+ <supported_graphics_clock>923 MHz</supported_graphics_clock>
+ <supported_graphics_clock>911 MHz</supported_graphics_clock>
+ <supported_graphics_clock>898 MHz</supported_graphics_clock>
+ <supported_graphics_clock>885 MHz</supported_graphics_clock>
+ <supported_graphics_clock>873 MHz</supported_graphics_clock>
+ <supported_graphics_clock>860 MHz</supported_graphics_clock>
+ <supported_graphics_clock>847 MHz</supported_graphics_clock>
+ <supported_graphics_clock>835 MHz</supported_graphics_clock>
+ <supported_graphics_clock>822 MHz</supported_graphics_clock>
+ <supported_graphics_clock>810 MHz</supported_graphics_clock>
+ <supported_graphics_clock>797 MHz</supported_graphics_clock>
+ <supported_graphics_clock>784 MHz</supported_graphics_clock>
+ <supported_graphics_clock>772 MHz</supported_graphics_clock>
+ <supported_graphics_clock>759 MHz</supported_graphics_clock>
+ <supported_graphics_clock>746 MHz</supported_graphics_clock>
+ <supported_graphics_clock>734 MHz</supported_graphics_clock>
+ <supported_graphics_clock>721 MHz</supported_graphics_clock>
+ <supported_graphics_clock>708 MHz</supported_graphics_clock>
+ <supported_graphics_clock>696 MHz</supported_graphics_clock>
+ <supported_graphics_clock>683 MHz</supported_graphics_clock>
+ <supported_graphics_clock>670 MHz</supported_graphics_clock>
+ <supported_graphics_clock>658 MHz</supported_graphics_clock>
+ <supported_graphics_clock>645 MHz</supported_graphics_clock>
+ <supported_graphics_clock>632 MHz</supported_graphics_clock>
+ <supported_graphics_clock>620 MHz</supported_graphics_clock>
+ <supported_graphics_clock>607 MHz</supported_graphics_clock>
+ <supported_graphics_clock>594 MHz</supported_graphics_clock>
+ <supported_graphics_clock>582 MHz</supported_graphics_clock>
+ <supported_graphics_clock>569 MHz</supported_graphics_clock>
+ <supported_graphics_clock>556 MHz</supported_graphics_clock>
+ <supported_graphics_clock>544 MHz</supported_graphics_clock>
+ </supported_mem_clock>
+ </supported_clocks>
+ <processes>
+ </processes>
+ <accounted_processes>
+ </accounted_processes>
+ </gpu>
+
+</nvidia_smi_log>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HADOOP-14912. FairCallQueue may
defer servicing calls. Contributed by Daryn Sharp
Posted by su...@apache.org.
HADOOP-14912. FairCallQueue may defer servicing calls. Contributed by Daryn Sharp
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1123f8f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1123f8f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1123f8f0
Branch: refs/heads/YARN-5881
Commit: 1123f8f0b62292197f5433cd40e66d8620044608
Parents: f29e55b
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Oct 10 14:47:25 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Oct 10 14:47:25 2017 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/FairCallQueue.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1123f8f0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 20161b8..6d9ea3e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -122,13 +122,15 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
private E removeNextElement() {
int priority = multiplexer.getAndAdvanceCurrentIndex();
E e = queues.get(priority).poll();
- if (e == null) {
+ // a semaphore permit has been acquired, so an element MUST be extracted
+ // or the semaphore and queued elements will go out of sync. loop to
+ // avoid race condition if elements are added behind the current position,
+ // awakening other threads that poll the elements ahead of our position.
+ while (e == null) {
for (int idx = 0; e == null && idx < queues.size(); idx++) {
e = queues.get(idx).poll();
}
}
- // guaranteed to find an element if caller acquired permit.
- assert e != null : "consumer didn't acquire semaphore!";
return e;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDFS-12599. Remove Mockito
dependency from DataNodeTestUtils. Contributed by Ted Yu.
Posted by su...@apache.org.
HDFS-12599. Remove Mockito dependency from DataNodeTestUtils. Contributed by Ted Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09ad848b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09ad848b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09ad848b
Branch: refs/heads/YARN-5881
Commit: 09ad848b6484c4d36b2d59903f9ec370fdb18d3e
Parents: 4664431
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Oct 9 10:29:01 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Oct 9 10:29:01 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/DataNodeTestUtils.java | 32 +++-----------------
.../datanode/InternalDataNodeTestUtils.java | 27 +++++++++++++++++
.../server/datanode/TestBlockReplacement.java | 2 +-
.../hadoop/hdfs/server/mover/TestMover.java | 6 ++--
4 files changed, 36 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ad848b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
index 6d5ab71..19d9dfc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
@@ -37,16 +36,16 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
import org.apache.hadoop.test.GenericTestUtils;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import com.google.common.base.Supplier;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
+
+/**
+ * DO NOT ADD MOCKITO IMPORTS HERE Or Downstream projects may not
+ * be able to start mini dfs cluster. THANKS.
+ */
/**
* Utility class for accessing package-private DataNode information during tests.
@@ -197,27 +196,6 @@ public class DataNodeTestUtils {
}
/**
- * This method is used to mock the data node block pinning API.
- *
- * @param dn datanode
- * @param pinned true if the block is pinned, false otherwise
- * @throws IOException
- */
- public static void mockDatanodeBlkPinning(final DataNode dn,
- final boolean pinned) throws IOException {
- final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
- dn.data = Mockito.spy(data);
-
- doAnswer(new Answer<Object>() {
- public Object answer(InvocationOnMock invocation) throws IOException {
- // Bypass the argument to FsDatasetImpl#getPinning to show that
- // the block is pinned.
- return pinned;
- }
- }).when(dn.data).getPinning(any(ExtendedBlock.class));
- }
-
- /**
* Reconfigure a DataNode by setting a new list of volumes.
*
* @param dn DataNode to reconfigure
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ad848b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
index 876a854..b453991 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
@@ -32,7 +32,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -49,6 +52,9 @@ import org.mockito.stubbing.Answer;
import com.google.common.base.Preconditions;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+
/**
* An internal-facing only collection of test utilities for the DataNode. This
* is to ensure that test-scope dependencies aren't inadvertently leaked
@@ -65,6 +71,27 @@ public class InternalDataNodeTestUtils {
}
/**
+ * This method is used to mock the data node block pinning API.
+ *
+ * @param dn datanode
+ * @param pinned true if the block is pinned, false otherwise
+ * @throws IOException
+ */
+ public static void mockDatanodeBlkPinning(final DataNode dn,
+ final boolean pinned) throws IOException {
+ final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
+ dn.data = Mockito.spy(data);
+
+ doAnswer(new Answer<Object>() {
+ public Object answer(InvocationOnMock invocation) throws IOException {
+ // Bypass the argument to FsDatasetImpl#getPinning to show that
+ // the block is pinned.
+ return pinned;
+ }
+ }).when(dn.data).getPinning(any(ExtendedBlock.class));
+ }
+
+ /**
* Insert a Mockito spy object between the given DataNode and
* the given NameNode. This can be used to delay or wait for
* RPC calls on the datanode->NN path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ad848b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
index 97255ae..f75b23d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
@@ -252,7 +252,7 @@ public class TestBlockReplacement {
for (int i = 0; i < cluster.getDataNodes().size(); i++) {
DataNode dn = cluster.getDataNodes().get(i);
LOG.info("Simulate block pinning in datanode " + dn);
- DataNodeTestUtils.mockDatanodeBlkPinning(dn, true);
+ InternalDataNodeTestUtils.mockDatanodeBlkPinning(dn, true);
}
// Block movement to a different datanode should fail as the block is
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ad848b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 8ff660f..461da07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -76,7 +76,7 @@ import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.mover.Mover.MLocation;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.http.HttpConfig;
@@ -758,7 +758,7 @@ public class TestMover {
for (int i = 0; i < cluster.getDataNodes().size(); i++) {
DataNode dn = cluster.getDataNodes().get(i);
LOG.info("Simulate block pinning in datanode {}", dn);
- DataNodeTestUtils.mockDatanodeBlkPinning(dn, true);
+ InternalDataNodeTestUtils.mockDatanodeBlkPinning(dn, true);
}
// move file blocks to ONE_SSD policy
@@ -896,7 +896,7 @@ public class TestMover {
if (dn.getDatanodeId().getDatanodeUuid()
.equals(datanodeInfo.getDatanodeUuid())) {
LOG.info("Simulate block pinning in datanode {}", datanodeInfo);
- DataNodeTestUtils.mockDatanodeBlkPinning(dn, true);
+ InternalDataNodeTestUtils.mockDatanodeBlkPinning(dn, true);
break;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: Revert "HDFS-12603. Enable async
edit logging by default."
Posted by su...@apache.org.
Revert "HDFS-12603. Enable async edit logging by default."
This reverts commit afb42aeabf1317b755ab79e0265bc90920c896ac.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82cd85d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82cd85d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82cd85d8
Branch: refs/heads/YARN-5881
Commit: 82cd85d8680912b8d268c875bd51598b20f3313c
Parents: f9ff17f
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Oct 9 17:53:56 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Oct 9 17:53:56 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 7 +------
.../hdfs/server/namenode/FSEditLogAsync.java | 5 -----
.../src/main/resources/hdfs-default.xml | 2 +-
.../namenode/ha/TestFailureToReadEdits.java | 22 +++++++-------------
5 files changed, 10 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cd85d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 395b192..1f96763 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
- public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
+ public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cd85d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 7d19b04..a8f5bfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
*
* In a non-HA setup:
*
- * The log starts in UNINITIALIZED state upon construction. Once it's
+ * The log starts in UNITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
@@ -1837,9 +1837,4 @@ public class FSEditLog implements LogsPurgeable {
}
return count;
}
-
- @Override
- public String toString() {
- return super.toString();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cd85d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
index 1f5dc75..c14a310 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
@@ -319,9 +319,4 @@ class FSEditLogAsync extends FSEditLog implements Runnable {
return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]";
}
}
-
- @Override
- public String toString() {
- return super.toString();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cd85d8/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3752578..3491ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4105,7 +4105,7 @@
<property>
<name>dfs.namenode.edits.asynclogging</name>
- <value>true</value>
+ <value>false</value>
<description>
If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82cd85d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index 2bc246d..93c717c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -75,7 +75,6 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random();
private final TestType clusterType;
- private final boolean useAsyncEditLogging;
private Configuration conf;
private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@@ -89,23 +88,18 @@ public class TestFailureToReadEdits {
};
/**
- * Run this suite of tests for {QJM-based, file-based HA} x {async
- * edit logging enabled, disabled}.
+ * Run this suite of tests both for QJM-based HA and for file-based
+ * HA.
*/
@Parameters
public static Iterable<Object[]> data() {
- return Arrays.asList(new Object[][]{
- {TestType.SHARED_DIR_HA, Boolean.FALSE},
- {TestType.SHARED_DIR_HA, Boolean.TRUE},
- {TestType.QJM_HA, Boolean.FALSE},
- {TestType.QJM_HA, Boolean.TRUE},
- });
+ return Arrays.asList(new Object[][] {
+ { TestType.SHARED_DIR_HA },
+ { TestType.QJM_HA } });
}
-
- public TestFailureToReadEdits(TestType clusterType, Boolean
- useAsyncEditLogging) {
+
+ public TestFailureToReadEdits(TestType clusterType) {
this.clusterType = clusterType;
- this.useAsyncEditLogging = useAsyncEditLogging;
}
@Before
@@ -115,8 +109,6 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
- useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDFS-12573. Divide the total
blocks metrics into replicated and erasure coded. Contributed by Takanobu
Asanuma.
Posted by su...@apache.org.
HDFS-12573. Divide the total blocks metrics into replicated and erasure coded. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78af6cdc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78af6cdc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78af6cdc
Branch: refs/heads/YARN-5881
Commit: 78af6cdc5359404139665d81447f28d26b7bb43b
Parents: 1123f8f
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Tue Oct 10 14:23:29 2017 -0700
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Tue Oct 10 14:23:29 2017 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 10 ++
.../hdfs/server/blockmanagement/BlocksMap.java | 40 ++++++-
.../hdfs/server/namenode/FSNamesystem.java | 12 +++
.../namenode/metrics/ECBlockGroupsMBean.java | 5 +
.../namenode/metrics/ReplicatedBlocksMBean.java | 5 +
.../server/namenode/TestNameNodeMXBean.java | 107 ++++++++++++++++++-
6 files changed, 177 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78af6cdc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 0545bb2..386ba27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -257,6 +257,11 @@ public class BlockManager implements BlockStatsMXBean {
}
/** Used by metrics. */
+ public long getTotalReplicatedBlocks() {
+ return blocksMap.getReplicatedBlocks();
+ }
+
+ /** Used by metrics. */
public long getLowRedundancyECBlockGroups() {
return neededReconstruction.getLowRedundancyECBlockGroups();
}
@@ -276,6 +281,11 @@ public class BlockManager implements BlockStatsMXBean {
return invalidateBlocks.getECBlocks();
}
+ /** Used by metrics. */
+ public long getTotalECBlockGroups() {
+ return blocksMap.getECBlockGroups();
+ }
+
/**
* redundancyRecheckInterval is how often namenode checks for new
* reconstruction work.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78af6cdc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index f7cde90..6f13da9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Collections;
import java.util.Iterator;
+import java.util.concurrent.atomic.LongAdder;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
@@ -37,6 +38,9 @@ class BlocksMap {
private GSet<Block, BlockInfo> blocks;
+ private final LongAdder totalReplicatedBlocks = new LongAdder();
+ private final LongAdder totalECBlockGroups = new LongAdder();
+
BlocksMap(int capacity) {
// Use 2% of total memory to size the GSet capacity
this.capacity = capacity;
@@ -65,6 +69,8 @@ class BlocksMap {
void clear() {
if (blocks != null) {
blocks.clear();
+ totalReplicatedBlocks.reset();
+ totalECBlockGroups.reset();
}
}
@@ -76,6 +82,7 @@ class BlocksMap {
if (info != b) {
info = b;
blocks.put(info);
+ incrementBlockStat(info);
}
info.setBlockCollectionId(bc.getId());
return info;
@@ -88,8 +95,10 @@ class BlocksMap {
*/
void removeBlock(Block block) {
BlockInfo blockInfo = blocks.remove(block);
- if (blockInfo == null)
+ if (blockInfo == null) {
return;
+ }
+ decrementBlockStat(block);
assert blockInfo.getBlockCollectionId() == INodeId.INVALID_INODE_ID;
final int size = blockInfo.isStriped() ?
@@ -166,6 +175,7 @@ class BlocksMap {
if (info.hasNoStorage() // no datanodes left
&& info.isDeleted()) { // does not belong to a file
blocks.remove(b); // remove block from the map
+ decrementBlockStat(b);
}
return removed;
}
@@ -196,4 +206,32 @@ class BlocksMap {
int getCapacity() {
return capacity;
}
+
+ private void incrementBlockStat(Block block) {
+ if (BlockIdManager.isStripedBlockID(block.getBlockId())) {
+ totalECBlockGroups.increment();
+ } else {
+ totalReplicatedBlocks.increment();
+ }
+ }
+
+ private void decrementBlockStat(Block block) {
+ if (BlockIdManager.isStripedBlockID(block.getBlockId())) {
+ totalECBlockGroups.decrement();
+ assert totalECBlockGroups.longValue() >= 0 :
+ "Total number of ec block groups should be non-negative";
+ } else {
+ totalReplicatedBlocks.decrement();
+ assert totalReplicatedBlocks.longValue() >= 0 :
+ "Total number of replicated blocks should be non-negative";
+ }
+ }
+
+ long getReplicatedBlocks() {
+ return totalReplicatedBlocks.longValue();
+ }
+
+ long getECBlockGroups() {
+ return totalECBlockGroups.longValue();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78af6cdc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index dc50808..e8d7161 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4719,6 +4719,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return blockManager.getPendingDeletionReplicatedBlocks();
}
+ @Override // ReplicatedBlocksMBean
+ @Metric({"TotalReplicatedBlocks", "Total number of replicated blocks"})
+ public long getTotalReplicatedBlocks() {
+ return blockManager.getTotalReplicatedBlocks();
+ }
+
@Override // ECBlockGroupsMBean
@Metric({"LowRedundancyECBlockGroups", "Number of erasure coded block " +
"groups with low redundancy"})
@@ -4754,6 +4760,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return blockManager.getPendingDeletionECBlocks();
}
+ @Override // ECBlockGroupsMBean
+ @Metric({"TotalECBlockGroups", "Total number of erasure coded block groups"})
+ public long getTotalECBlockGroups() {
+ return blockManager.getTotalECBlockGroups();
+ }
+
@Override
public long getBlockDeletionStartTime() {
return startTime + blockManager.getStartupDelayBlockDeletionInMs();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78af6cdc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
index 474f3ed..808fc62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
@@ -56,4 +56,9 @@ public interface ECBlockGroupsMBean {
* Return count of erasure coded blocks that are pending deletion.
*/
long getPendingDeletionECBlocks();
+
+ /**
+ * Return total number of erasure coded block groups.
+ */
+ long getTotalECBlockGroups();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78af6cdc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ReplicatedBlocksMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ReplicatedBlocksMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ReplicatedBlocksMBean.java
index e2c924e..a20dd4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ReplicatedBlocksMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ReplicatedBlocksMBean.java
@@ -60,4 +60,9 @@ public interface ReplicatedBlocksMBean {
* Return count of blocks that are pending deletion.
*/
long getPendingDeletionReplicatedBlocks();
+
+ /**
+ * Return total number of replicated blocks.
+ */
+ long getTotalReplicatedBlocks();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78af6cdc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 93a7525..9c165d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -731,7 +731,8 @@ public class TestNameNodeMXBean {
DistributedFileSystem fs = null;
try {
Configuration conf = new HdfsConfiguration();
- int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
+ int dataBlocks = StripedFileTestUtil.getDefaultECPolicy()
+ .getNumDataUnits();
int parityBlocks =
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
@@ -860,4 +861,108 @@ public class TestNameNodeMXBean {
}
}
}
+
+ @Test
+ public void testTotalBlocksMetrics() throws Exception {
+ MiniDFSCluster cluster = null;
+ FSNamesystem namesystem = null;
+ DistributedFileSystem fs = null;
+ try {
+ Configuration conf = new HdfsConfiguration();
+ int dataBlocks = StripedFileTestUtil.getDefaultECPolicy()
+ .getNumDataUnits();
+ int parityBlocks =
+ StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
+ int totalSize = dataBlocks + parityBlocks;
+ int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
+ int stripesPerBlock = 2;
+ int blockSize = stripesPerBlock * cellSize;
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(totalSize).build();
+ namesystem = cluster.getNamesystem();
+ fs = cluster.getFileSystem();
+ fs.enableErasureCodingPolicy(
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+ verifyTotalBlocksMetrics(0L, 0L, namesystem.getTotalBlocks());
+
+ // create small file
+ Path replDirPath = new Path("/replicated");
+ Path replFileSmall = new Path(replDirPath, "replfile_small");
+ final short factor = 3;
+ DFSTestUtil.createFile(fs, replFileSmall, blockSize, factor, 0);
+ DFSTestUtil.waitReplication(fs, replFileSmall, factor);
+
+ Path ecDirPath = new Path("/striped");
+ fs.mkdir(ecDirPath, FsPermission.getDirDefault());
+ fs.getClient().setErasureCodingPolicy(ecDirPath.toString(),
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+ Path ecFileSmall = new Path(ecDirPath, "ecfile_small");
+ final int smallLength = cellSize * dataBlocks;
+ final byte[] smallBytes = StripedFileTestUtil.generateBytes(smallLength);
+ DFSTestUtil.writeFile(fs, ecFileSmall, smallBytes);
+ verifyTotalBlocksMetrics(1L, 1L, namesystem.getTotalBlocks());
+
+ // create learge file
+ Path replFileLarge = new Path(replDirPath, "replfile_large");
+ DFSTestUtil.createFile(fs, replFileLarge, 2 * blockSize, factor, 0);
+ DFSTestUtil.waitReplication(fs, replFileLarge, factor);
+
+ Path ecFileLarge = new Path(ecDirPath, "ecfile_large");
+ final int largeLength = blockSize * totalSize + smallLength;
+ final byte[] largeBytes = StripedFileTestUtil.generateBytes(largeLength);
+ DFSTestUtil.writeFile(fs, ecFileLarge, largeBytes);
+ verifyTotalBlocksMetrics(3L, 3L, namesystem.getTotalBlocks());
+
+ // delete replicated files
+ fs.delete(replDirPath, true);
+ verifyTotalBlocksMetrics(0L, 3L, namesystem.getTotalBlocks());
+
+ // delete ec files
+ fs.delete(ecDirPath, true);
+ verifyTotalBlocksMetrics(0L, 0L, namesystem.getTotalBlocks());
+ } finally {
+ if (fs != null) {
+ try {
+ fs.close();
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+ if (namesystem != null) {
+ try {
+ namesystem.close();
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ void verifyTotalBlocksMetrics(long expectedTotalReplicatedBlocks,
+ long expectedTotalECBlockGroups, long actualTotalBlocks)
+ throws Exception {
+ long expectedTotalBlocks = expectedTotalReplicatedBlocks
+ + expectedTotalECBlockGroups;
+ assertEquals("Unexpected total blocks!", expectedTotalBlocks,
+ actualTotalBlocks);
+
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName replStateMBeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=ReplicatedBlocksState");
+ ObjectName ecBlkGrpStateMBeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=ECBlockGroupsState");
+ Long totalReplicaBlocks = (Long) mbs.getAttribute(replStateMBeanName,
+ "TotalReplicatedBlocks");
+ Long totalECBlockGroups = (Long) mbs.getAttribute(ecBlkGrpStateMBeanName,
+ "TotalECBlockGroups");
+ assertEquals("Unexpected total replicated blocks!",
+ expectedTotalReplicatedBlocks, totalReplicaBlocks.longValue());
+ assertEquals("Unexpected total ec block groups!",
+ expectedTotalECBlockGroups, totalECBlockGroups.longValue());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HADOOP-13556. Change
Configuration.getPropsWithPrefix to use getProps instead of iterator. (Larry
McCay via asuresh)
Posted by su...@apache.org.
HADOOP-13556. Change Configuration.getPropsWithPrefix to use getProps instead of iterator. (Larry McCay via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6c2c905
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6c2c905
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6c2c905
Branch: refs/heads/YARN-5881
Commit: b6c2c9058e83116dcca46cd6934db3428f931347
Parents: 8acdf5c
Author: Arun Suresh <as...@apache.org>
Authored: Wed Oct 11 15:21:21 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Oct 11 15:21:21 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/conf/Configuration.java | 9 ++++++---
.../org/apache/hadoop/conf/TestConfiguration.java | 15 +++++++++++++++
2 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6c2c905/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 2890853..9d5bb1b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2700,11 +2700,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @return mapping of configuration properties with prefix stripped
*/
public Map<String, String> getPropsWithPrefix(String confPrefix) {
+ Properties props = getProps();
+ Enumeration e = props.propertyNames();
Map<String, String> configMap = new HashMap<>();
- for (Map.Entry<String, String> entry : this) {
- String name = entry.getKey();
+ String name = null;
+ while (e.hasMoreElements()) {
+ name = (String) e.nextElement();
if (name.startsWith(confPrefix)) {
- String value = this.get(name);
+ String value = props.getProperty(name);
name = name.substring(confPrefix.length());
configMap.put(name, value);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6c2c905/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index a806b8c..52215da 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -2242,6 +2242,21 @@ public class TestConfiguration {
FileUtil.fullyDelete(tmpDir);
}
+ public void testGettingPropertiesWithPrefix() throws Exception {
+ Configuration conf = new Configuration();
+ for (int i = 0; i < 10; i++) {
+ conf.set("prefix" + ".name" + i, "value");
+ }
+ conf.set("different.prefix" + ".name", "value");
+ Map<String, String> props = conf.getPropsWithPrefix("prefix");
+ assertEquals(props.size(), 10);
+
+ // test call with no properties for a given prefix
+ props = conf.getPropsWithPrefix("none");
+ assertNotNull(props.isEmpty());
+ assertTrue(props.isEmpty());
+ }
+
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-5951. Changes to allow
CapacityScheduler to use configuration store
Posted by su...@apache.org.
YARN-5951. Changes to allow CapacityScheduler to use configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef59cbe0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef59cbe0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef59cbe0
Branch: refs/heads/YARN-5881
Commit: ef59cbe08aabe9844159cb4192d6345e4e3caab2
Parents: 09ad848
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Jan 30 19:03:48 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:22 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 37 +++++------
.../CapacitySchedulerConfiguration.java | 9 +++
.../capacity/conf/CSConfigurationProvider.java | 46 ++++++++++++++
.../conf/FileBasedCSConfigurationProvider.java | 67 ++++++++++++++++++++
.../scheduler/capacity/conf/package-info.java | 29 +++++++++
.../capacity/TestCapacityScheduler.java | 4 +-
6 files changed, 170 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef59cbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 7f50272..e74cbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -103,6 +102,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -167,6 +168,8 @@ public class CapacityScheduler extends
private int maxAssignPerHeartbeat;
+ private CSConfigurationProvider csConfProvider;
+
@Override
public void setConf(Configuration conf) {
yarnConf = conf;
@@ -289,7 +292,18 @@ public class CapacityScheduler extends
IOException {
try {
writeLock.lock();
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ String confProviderStr = configuration.get(
+ CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ if (confProviderStr.equals(
+ CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
+ this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
+ } else {
+ throw new IOException("Invalid CS configuration provider: " +
+ confProviderStr);
+ }
+ this.csConfProvider.init(configuration);
+ this.conf = this.csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
this.minimumAllocation = super.getMinimumAllocation();
initMaximumResourceCapability(super.getMaximumAllocation());
@@ -399,7 +413,7 @@ public class CapacityScheduler extends
writeLock.lock();
Configuration configuration = new Configuration(newConf);
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ this.conf = csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -1831,23 +1845,6 @@ public class CapacityScheduler extends
return true;
}
- private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
- Configuration configuration) throws IOException {
- try {
- InputStream CSInputStream =
- this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE);
- if (CSInputStream != null) {
- configuration.addResource(CSInputStream);
- return new CapacitySchedulerConfiguration(configuration, false);
- }
- return new CapacitySchedulerConfiguration(configuration, true);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
private String getDefaultReservationQueueName(String planQueueName) {
return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef59cbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3a519ec..3821e24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -315,6 +315,15 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
+
+ public static final String CS_CONF_PROVIDER = PREFIX
+ + "configuration.provider";
+
+ @Private
+ public static final String FILE_CS_CONF_PROVIDER = "file";
+
+ @Private
+ public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef59cbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
new file mode 100644
index 0000000..c9984ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+
+/**
+ * Configuration provider for {@link CapacityScheduler}.
+ */
+public interface CSConfigurationProvider {
+
+ /**
+ * Initialize the configuration provider with given conf.
+ * @param conf configuration to initialize with
+ */
+ void init(Configuration conf);
+
+ /**
+ * Loads capacity scheduler configuration object.
+ * @param conf initial bootstrap configuration
+ * @return CS configuration
+ * @throws IOException if fail to retrieve configuration
+ */
+ CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef59cbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
new file mode 100644
index 0000000..51c64fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * {@link CapacityScheduler} configuration provider based on local
+ * {@code capacity-scheduler.xml} file.
+ */
+public class FileBasedCSConfigurationProvider implements
+ CSConfigurationProvider {
+
+ private RMContext rmContext;
+
+ /**
+ * Construct file based CS configuration provider with given context.
+ * @param rmContext the RM context
+ */
+ public FileBasedCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration conf) {}
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException {
+ try {
+ InputStream csInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (csInputStream != null) {
+ conf.addResource(csInputStream);
+ return new CapacitySchedulerConfiguration(conf, false);
+ }
+ return new CapacitySchedulerConfiguration(conf, true);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef59cbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
new file mode 100644
index 0000000..08d0522
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package
+ * org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf
+ * contains classes related to capacity scheduler configuration management.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef59cbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 8440519..1dea4ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -241,13 +241,13 @@ public class TestCapacityScheduler {
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
- ResourceScheduler scheduler = new CapacityScheduler();
+ CapacityScheduler scheduler = new CapacityScheduler();
scheduler.setRMContext(resourceManager.getRMContext());
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, mockContext);
+ scheduler.init(conf);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-7205. Log improvements for
the ResourceUtils. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7205. Log improvements for the ResourceUtils. (Sunil G via wangda)
Change-Id: I0f5b7a7f68ec5d3e1d52211f83fdd089bc0bfd37
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bcc49e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bcc49e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bcc49e6
Branch: refs/heads/YARN-5881
Commit: 8bcc49e6771ca75f012211e27870a421b19233e7
Parents: b6c2c90
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Oct 11 15:25:28 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Oct 11 15:25:28 2017 -0700
----------------------------------------------------------------------
.../yarn/api/records/ResourceTypeInfo.java | 7 +--
.../yarn/util/resource/ResourceUtils.java | 45 +++++++++++---------
2 files changed, 30 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bcc49e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java
index b6f7f14..8775342 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java
@@ -152,9 +152,10 @@ public abstract class ResourceTypeInfo implements Comparable<ResourceTypeInfo> {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("<name=").append(this.getName()).append(" default-unit=").append(
- this.getDefaultUnit()).append(" type=" + getResourceType()).append(
- ">");
+ sb.append(this.getName());
+ if (!this.getDefaultUnit().isEmpty()) {
+ sb.append(" (unit=").append(this.getDefaultUnit()).append(")");
+ }
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bcc49e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index a1b5f6e..1779899 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -22,8 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -118,19 +116,21 @@ public class ResourceUtils {
Map<String, ResourceInformation> res) {
ResourceInformation ri;
if (!res.containsKey(MEMORY)) {
- LOG.info("Adding resource type - name = " + MEMORY + ", units = "
- + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
- + ResourceTypes.COUNTABLE);
- ri = ResourceInformation
- .newInstance(MEMORY,
- ResourceInformation.MEMORY_MB.getUnits());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding resource type - name = " + MEMORY + ", units = "
+ + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
+ + ResourceTypes.COUNTABLE);
+ }
+ ri = ResourceInformation.newInstance(MEMORY,
+ ResourceInformation.MEMORY_MB.getUnits());
res.put(MEMORY, ri);
}
if (!res.containsKey(VCORES)) {
- LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
- + ResourceTypes.COUNTABLE);
- ri =
- ResourceInformation.newInstance(VCORES);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding resource type - name = " + VCORES
+ + ", units = , type = " + ResourceTypes.COUNTABLE);
+ }
+ ri = ResourceInformation.newInstance(VCORES);
res.put(VCORES, ri);
}
}
@@ -343,11 +343,11 @@ public class ResourceUtils {
}
try {
addResourcesFileToConf(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
} catch (FileNotFoundException fe) {
- LOG.debug("Unable to find '" + resourceFile + "'.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Unable to find '" + resourceFile + "'.");
+ }
}
-
initializeResourcesMap(conf);
}
}
@@ -388,7 +388,9 @@ public class ResourceUtils {
Configuration conf) throws FileNotFoundException {
try {
InputStream ris = getConfInputStream(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found " + resourceFile + ", adding to configuration");
+ }
conf.addResource(ris);
} catch (FileNotFoundException fe) {
throw fe;
@@ -472,7 +474,10 @@ public class ResourceUtils {
}
}
} catch (FileNotFoundException fe) {
- LOG.info("Couldn't find node resources file");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Couldn't find node resources file: "
+ + YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE);
+ }
}
return nodeResources;
}
@@ -492,8 +497,10 @@ public class ResourceUtils {
Long.valueOf(value.substring(0, value.length() - units.length()));
nodeResources.get(resourceType).setValue(resourceValue);
nodeResources.get(resourceType).setUnits(units);
- LOG.debug("Setting value for resource type " + resourceType + " to "
- + resourceValue + " with units " + units);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting value for resource type " + resourceType + " to "
+ + resourceValue + " with units " + units);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HADOOP-14910. Upgrade netty-all
jar to latest 4.0.x.Final. Contributed by Vinayakumar B.
Posted by su...@apache.org.
HADOOP-14910. Upgrade netty-all jar to latest 4.0.x.Final. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f29e55bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f29e55bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f29e55bf
Branch: refs/heads/YARN-5881
Commit: f29e55bf5e634b3be28c9e0e993f78877528b5c2
Parents: d6602b5
Author: Jitendra Pandey <ji...@apache.org>
Authored: Tue Oct 10 11:00:07 2017 -0700
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Tue Oct 10 11:00:07 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java | 3 +--
hadoop-project/pom.xml | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29e55bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index bb28110..16380e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -59,8 +59,7 @@ class ParameterParser {
private final Map<String, List<String>> params;
ParameterParser(QueryStringDecoder decoder, Configuration conf) {
- this.path = decodeComponent(decoder.path().substring(WEBHDFS_PREFIX_LENGTH),
- StandardCharsets.UTF_8);
+ this.path = decoder.path().substring(WEBHDFS_PREFIX_LENGTH);
this.params = decoder.parameters();
this.conf = conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f29e55bf/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 66d2eb4..8980f0e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -749,7 +749,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.0.23.Final</version>
+ <version>4.0.52.Final</version>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-6620. Add support in
NodeManager to isolate GPU devices by using CGroups. Contributed by Wangda
Tan.
Posted by su...@apache.org.
YARN-6620. Add support in NodeManager to isolate GPU devices by using CGroups. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa5cfc68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa5cfc68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa5cfc68
Branch: refs/heads/YARN-5881
Commit: fa5cfc68f37c78b6cf26ce13247b9ff34da806cd
Parents: 3de5744
Author: Sunil G <su...@apache.org>
Authored: Wed Oct 11 23:44:33 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Oct 11 23:44:33 2017 +0530
----------------------------------------------------------------------
.../yarn/api/records/ResourceInformation.java | 10 +
.../hadoop/yarn/conf/YarnConfiguration.java | 33 ++
.../yarn/util/resource/ResourceUtils.java | 51 +-
.../src/main/resources/yarn-default.xml | 42 +-
.../yarn/util/resource/TestResourceUtils.java | 17 +
.../server/nodemanager/ContainerExecutor.java | 3 +-
.../hadoop/yarn/server/nodemanager/Context.java | 3 +
.../nodemanager/DefaultContainerExecutor.java | 2 +-
.../nodemanager/LinuxContainerExecutor.java | 10 +-
.../yarn/server/nodemanager/NodeManager.java | 92 ++--
.../nodemanager/NodeStatusUpdaterImpl.java | 38 +-
.../linux/privileged/PrivilegedOperation.java | 1 +
.../linux/resources/ResourceHandlerChain.java | 4 +-
.../linux/resources/ResourceHandlerModule.java | 42 +-
.../resources/gpu/GpuResourceAllocator.java | 242 ++++++++
.../resources/gpu/GpuResourceHandlerImpl.java | 153 ++++++
.../NodeResourceUpdaterPlugin.java | 52 ++
.../resourceplugin/ResourcePlugin.java | 83 +++
.../resourceplugin/ResourcePluginManager.java | 106 ++++
.../resourceplugin/gpu/GpuDiscoverer.java | 254 +++++++++
.../gpu/GpuNodeResourceUpdateHandler.java | 66 +++
.../resourceplugin/gpu/GpuResourcePlugin.java | 61 +++
.../webapp/dao/gpu/GpuDeviceInformation.java | 72 +++
.../dao/gpu/GpuDeviceInformationParser.java | 87 +++
.../webapp/dao/gpu/PerGpuDeviceInformation.java | 165 ++++++
.../webapp/dao/gpu/PerGpuMemoryUsage.java | 58 ++
.../webapp/dao/gpu/PerGpuTemperature.java | 80 +++
.../webapp/dao/gpu/PerGpuUtilizations.java | 50 ++
.../server/nodemanager/NodeManagerTestBase.java | 164 ++++++
.../TestDefaultContainerExecutor.java | 4 +-
.../nodemanager/TestLinuxContainerExecutor.java | 2 +-
.../TestLinuxContainerExecutorWithMocks.java | 2 +-
.../server/nodemanager/TestNodeManager.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 100 +---
.../amrmproxy/BaseAMRMProxyTest.java | 46 +-
.../resources/TestResourceHandlerModule.java | 8 +-
.../resources/gpu/TestGpuResourceHandler.java | 382 +++++++++++++
.../TestContainersMonitorResourceChange.java | 2 +-
.../TestResourcePluginManager.java | 261 +++++++++
.../resourceplugin/gpu/TestGpuDiscoverer.java | 123 +++++
.../dao/gpu/TestGpuDeviceInformationParser.java | 50 ++
.../test/resources/nvidia-smi-sample-xml-output | 547 +++++++++++++++++++
42 files changed, 3365 insertions(+), 205 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 2a04094..e7a76e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.yarn.api.records;
+import com.google.common.collect.ImmutableMap;
import org.apache.curator.shaded.com.google.common.reflect.ClassPath;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+import java.util.Map;
+
/**
* Class to encapsulate information about a Resource - the name of the resource,
* the units(milli, micro, etc), the type(countable), and the value.
@@ -36,13 +39,20 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
private long minimumAllocation;
private long maximumAllocation;
+ // Known resource types
public static final String MEMORY_URI = "memory-mb";
public static final String VCORES_URI = "vcores";
+ public static final String GPU_URI = "yarn.io/gpu";
public static final ResourceInformation MEMORY_MB =
ResourceInformation.newInstance(MEMORY_URI, "Mi");
public static final ResourceInformation VCORES =
ResourceInformation.newInstance(VCORES_URI);
+ public static final ResourceInformation GPUS =
+ ResourceInformation.newInstance(GPU_URI);
+
+ public static final Map<String, ResourceInformation> MANDATORY_RESOURCES =
+ ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS);
/**
* Get the name for the resource.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ac8ff12..0d5f2cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1433,6 +1433,39 @@ public class YarnConfiguration extends Configuration {
public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT =
NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-yarn-mbit";
+ /**
+ * Prefix for computation resources, example of computation resources like
+ * GPU / FPGA / TPU, etc.
+ */
+ @Private
+ public static final String NM_RESOURCE_PLUGINS =
+ NM_PREFIX + "resource-plugins";
+
+ /**
+ * Prefix for gpu configurations. Work in progress: This configuration
+ * parameter may be changed/removed in the future.
+ */
+ @Private
+ public static final String NM_GPU_RESOURCE_PREFIX =
+ NM_RESOURCE_PLUGINS + ".gpu.";
+
+ @Private
+ public static final String NM_GPU_ALLOWED_DEVICES =
+ NM_GPU_RESOURCE_PREFIX + "allowed-gpu-devices";
+ @Private
+ public static final String AUTOMATICALLY_DISCOVER_GPU_DEVICES = "auto";
+
+ /**
+ * This setting controls where to how to invoke GPU binaries
+ */
+ @Private
+ public static final String NM_GPU_PATH_TO_EXEC =
+ NM_GPU_RESOURCE_PREFIX + "path-to-discovery-executables";
+
+ @Private
+ public static final String DEFAULT_NM_GPU_PATH_TO_EXEC = "";
+
+
/** NM Webapp address.**/
public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
public static final int DEFAULT_NM_WEBAPP_PORT = 8042;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 0564d74..a1b5f6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -46,6 +46,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
+
/**
* Helper class to read the resource-types to be supported by the system.
*/
@@ -82,33 +84,32 @@ public class ResourceUtils {
*/
String key = "memory";
if (resourceInformationMap.containsKey(key)) {
- LOG.warn("Attempt to define resource '" + key +
- "', but it is not allowed.");
- throw new YarnRuntimeException("Attempt to re-define mandatory resource '"
- + key + "'.");
+ LOG.warn(
+ "Attempt to define resource '" + key + "', but it is not allowed.");
+ throw new YarnRuntimeException(
+ "Attempt to re-define mandatory resource '" + key + "'.");
}
- if (resourceInformationMap.containsKey(MEMORY)) {
- ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
- String memUnits = ResourceInformation.MEMORY_MB.getUnits();
- ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
- if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
- .equals(memType)) {
- throw new YarnRuntimeException(
- "Attempt to re-define mandatory resource 'memory-mb'. It can only"
- + " be of type 'COUNTABLE' and have units 'Mi'.");
- }
- }
-
- if (resourceInformationMap.containsKey(VCORES)) {
- ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
- String vcoreUnits = ResourceInformation.VCORES.getUnits();
- ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
- if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
- .getResourceType().equals(vcoreType)) {
- throw new YarnRuntimeException(
- "Attempt to re-define mandatory resource 'vcores'. It can only be"
- + " of type 'COUNTABLE' and have units ''(no units).");
+ for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
+ ResourceInformation.MANDATORY_RESOURCES.entrySet()) {
+ key = mandatoryResourceEntry.getKey();
+ ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
+
+ ResourceInformation newDefinedRI = resourceInformationMap.get(key);
+ if (newDefinedRI != null) {
+ String expectedUnit = mandatoryRI.getUnits();
+ ResourceTypes expectedType = mandatoryRI.getResourceType();
+ String actualUnit = newDefinedRI.getUnits();
+ ResourceTypes actualType = newDefinedRI.getResourceType();
+
+ if (!expectedUnit.equals(actualUnit) || !expectedType.equals(
+ actualType)) {
+ throw new YarnRuntimeException("Defined mandatory resource type="
+ + key + " inside resource-types.xml, however its type or "
+ + "unit is conflict to mandatory resource types, expected type="
+ + expectedType + ", unit=" + expectedUnit + "; actual type="
+ + actualType + " actual unit=" + actualUnit);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 421bccf..4e78947 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3347,7 +3347,6 @@
<value>false</value>
</property>
-
<!-- resource types configuration -->
<property>
<name>yarn.resource-types</name>
@@ -3431,4 +3430,45 @@
<name>yarn.scheduler.configuration.zk-store.parent-path</name>
<value>/confstore</value>
</property>
+
+ <property>
+ <description>
+ When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified,
+ YARN NodeManager needs to run GPU discovery binary (now only support
+ nvidia-smi) to get GPU-related information.
+ When value is empty (default), YARN NodeManager will try to locate
+ discovery executable itself.
+ An example of the config value is: /usr/local/bin/nvidia-smi
+ </description>
+ <name>yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
+ Enable additional discovery/isolation of resources on the NodeManager,
+ split by comma. By default, this is empty. Acceptable values: { "yarn-io/gpu" }.
+ </description>
+ <name>yarn.nodemanager.resource-plugins</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
+ Specify GPU devices which can be managed by YARN NodeManager, split by comma
+ Number of GPU devices will be reported to RM to make scheduling decisions.
+ Set to auto (default) let YARN automatically discover GPU resource from
+ system.
+ Manually specify GPU devices if auto detect GPU device failed or admin
+ only want subset of GPU devices managed by YARN. GPU device is identified
+ by their minor device number. A common approach to get minor device number
+ of GPUs is using "nvidia-smi -q" and search "Minor Number" output. An
+ example of manual specification is "0,1,2,4" to allow YARN NodeManager
+ to manage GPU devices with minor number 0/1/2/4.
+ </description>
+ <name>yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices</name>
+ <value>auto</value>
+ </property>
+
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index a5550a7..7a68fb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -50,6 +50,23 @@ public class TestResourceUtils {
}
}
+ public static void addNewTypesToResources(String... resourceTypes) {
+ // Initialize resource map
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
+ riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
+
+ for (String newResource : resourceTypes) {
+ riMap.put(newResource, ResourceInformation
+ .newInstance(newResource, "", 0, ResourceTypes.COUNTABLE, 0,
+ Integer.MAX_VALUE));
+ }
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+ }
+
@Before
public void setup() {
ResourceUtils.resetResourceTypes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 0b4dbc9..7ee3e52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -113,9 +113,10 @@ public abstract class ContainerExecutor implements Configurable {
* Run the executor initialization steps.
* Verify that the necessary configs and permissions are in place.
*
+ * @param nmContext Context of NM
* @throws IOException if initialization fails
*/
- public abstract void init() throws IOException;
+ public abstract void init(Context nmContext) throws IOException;
/**
* This function localizes the JAR file on-demand.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index a2d00a4..a1c474f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManag
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
@@ -122,4 +123,6 @@ public interface Context {
ContainerExecutor getContainerExecutor();
ContainerStateTransitionListener getContainerStateTransitionListener();
+
+ ResourcePluginManager getResourcePluginManager();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 7c11d02..5d78f9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -135,7 +135,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
}
@Override
- public void init() throws IOException {
+ public void init(Context nmContext) throws IOException {
// nothing to do or verify here
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 9f1ec60..865d18d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -281,7 +282,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
- public void init() throws IOException {
+ public void init(Context nmContext) throws IOException {
Configuration conf = super.getConf();
// Send command to executor which will just start up,
@@ -305,7 +306,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
try {
resourceHandlerChain = ResourceHandlerModule
- .getConfiguredResourceHandlerChain(conf);
+ .getConfiguredResourceHandlerChain(conf, nmContext);
if (LOG.isDebugEnabled()) {
LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
!= null));
@@ -844,4 +845,9 @@ public class LinuxContainerExecutor extends ContainerExecutor {
e);
}
}
+
+ @VisibleForTesting
+ public ResourceHandler getResourceHandler() {
+ return resourceHandlerChain;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index a97b3f2..7a6106e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -18,23 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.state.MultiStateTransitionListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -65,12 +49,16 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ConfigurationNodeLabelsProvider;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
@@ -78,14 +66,25 @@ import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ScriptBasedNodeLabel
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMLeveldbStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
-import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
+import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.state.MultiStateTransitionListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
public class NodeManager extends CompositeService
implements EventHandler<NodeManagerEvent> {
@@ -331,6 +330,18 @@ public class NodeManager extends CompositeService
nmCheckintervalTime, scriptTimeout, scriptArgs);
}
+ @VisibleForTesting
+ protected ResourcePluginManager createResourcePluginManager() {
+ return new ResourcePluginManager();
+ }
+
+ @VisibleForTesting
+ protected ContainerExecutor createContainerExecutor(Configuration conf) {
+ return ReflectionUtils.newInstance(
+ conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
+ DefaultContainerExecutor.class, ContainerExecutor.class), conf);
+ }
+
@Override
protected void serviceInit(Configuration conf) throws Exception {
rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration
@@ -356,11 +367,20 @@ public class NodeManager extends CompositeService
this.aclsManager = new ApplicationACLsManager(conf);
- ContainerExecutor exec = ReflectionUtils.newInstance(
- conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
- DefaultContainerExecutor.class, ContainerExecutor.class), conf);
+ boolean isDistSchedulingEnabled =
+ conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
+ YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
+
+ this.context = createNMContext(containerTokenSecretManager,
+ nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
+
+ ResourcePluginManager pluginManager = createResourcePluginManager();
+ pluginManager.initialize(context);
+ ((NMContext)context).setResourcePluginManager(pluginManager);
+
+ ContainerExecutor exec = createContainerExecutor(conf);
try {
- exec.init();
+ exec.init(context);
} catch (IOException e) {
throw new YarnRuntimeException("Failed to initialize container executor", e);
}
@@ -376,13 +396,6 @@ public class NodeManager extends CompositeService
getNodeHealthScriptRunner(conf), dirsHandler);
addService(nodeHealthChecker);
- boolean isDistSchedulingEnabled =
- conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED,
- YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
-
- this.context = createNMContext(containerTokenSecretManager,
- nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
-
((NMContext)context).setContainerExecutor(exec);
@@ -459,6 +472,12 @@ public class NodeManager extends CompositeService
try {
super.serviceStop();
DefaultMetricsSystem.shutdown();
+
+ // Cleanup ResourcePluginManager
+ ResourcePluginManager rpm = context.getResourcePluginManager();
+ if (rpm != null) {
+ rpm.cleanup();
+ }
} finally {
// YARN-3641: NM's services stop get failed shouldn't block the
// release of NMLevelDBStore.
@@ -596,6 +615,8 @@ public class NodeManager extends CompositeService
private ContainerStateTransitionListener containerStateTransitionListener;
+ private ResourcePluginManager resourcePluginManager;
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
@@ -796,6 +817,15 @@ public class NodeManager extends CompositeService
ContainerStateTransitionListener transitionListener) {
this.containerStateTransitionListener = transitionListener;
}
+
+ public ResourcePluginManager getResourcePluginManager() {
+ return resourcePluginManager;
+ }
+
+ public void setResourcePluginManager(
+ ResourcePluginManager resourcePluginManager) {
+ this.resourcePluginManager = resourcePluginManager;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 3efe0bc..98e2e87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -33,6 +33,9 @@ import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -178,14 +181,15 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
long memoryMb = totalResource.getMemorySize();
float vMemToPMem =
conf.getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
long virtualMemoryMb = (long)Math.ceil(memoryMb * vMemToPMem);
-
int virtualCores = totalResource.getVirtualCores();
- LOG.info("Nodemanager resources: memory set to " + memoryMb + "MB.");
- LOG.info("Nodemanager resources: vcores set to " + virtualCores + ".");
- LOG.info("Nodemanager resources: " + totalResource);
+
+ // Update configured resources via plugins.
+ updateConfiguredResourcesViaPlugins(totalResource);
+
+ LOG.info("Nodemanager resources is set to: " + totalResource);
metrics.addResource(totalResource);
@@ -342,12 +346,27 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
return ServerRMProxy.createRMProxy(conf, ResourceTracker.class);
}
+ private void updateConfiguredResourcesViaPlugins(
+ Resource configuredResource) throws YarnException {
+ ResourcePluginManager pluginManager = context.getResourcePluginManager();
+ if (pluginManager != null && pluginManager.getNameToPlugins() != null) {
+ // Update configured resource
+ for (ResourcePlugin resourcePlugin : pluginManager.getNameToPlugins()
+ .values()) {
+ if (resourcePlugin.getNodeResourceHandlerInstance() != null) {
+ resourcePlugin.getNodeResourceHandlerInstance()
+ .updateConfiguredResource(configuredResource);
+ }
+ }
+ }
+ }
+
@VisibleForTesting
protected void registerWithRM()
throws YarnException, IOException {
RegisterNodeManagerResponse regNMResponse;
Set<NodeLabel> nodeLabels = nodeLabelsHandler.getNodeLabelsForRegistration();
-
+
// Synchronize NM-RM registration with
// ContainerManagerImpl#increaseContainersResource and
// ContainerManagerImpl#startContainers to avoid race condition
@@ -358,6 +377,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource,
nodeManagerVersionId, containerReports, getRunningApplications(),
nodeLabels, physicalResource);
+
if (containerReports != null) {
LOG.info("Registering with RM using containers :" + containerReports);
}
@@ -406,7 +426,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
if (masterKey != null) {
this.context.getContainerTokenSecretManager().setMasterKey(masterKey);
}
-
+
masterKey = regNMResponse.getNMTokenMasterKey();
if (masterKey != null) {
this.context.getNMTokenSecretManager().setMasterKey(masterKey);
@@ -732,7 +752,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
}
}
-
+
@Override
public long getRMIdentifier() {
return this.rmIdentifier;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
index 8402a16..db0b225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
@@ -51,6 +51,7 @@ public class PrivilegedOperation {
TC_READ_STATS("--tc-read-stats"),
ADD_PID_TO_CGROUP(""), //no CLI switch supported yet.
RUN_DOCKER_CMD("--run-docker"),
+ GPU("--module-gpu"),
LIST_AS_USER(""); //no CLI switch supported yet.
private final String option;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
index 955d216..72bf30c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -135,7 +136,8 @@ public class ResourceHandlerChain implements ResourceHandler {
return allOperations;
}
- List<ResourceHandler> getResourceHandlerList() {
+ @VisibleForTesting
+ public List<ResourceHandler> getResourceHandlerList() {
return Collections.unmodifiableList(resourceHandlers);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
index 3c61cd4..ce850ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
@@ -21,25 +21,28 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Provides mechanisms to get various resource handlers - cpu, memory, network,
@@ -206,22 +209,41 @@ public class ResourceHandlerModule {
}
private static void initializeConfiguredResourceHandlerChain(
- Configuration conf) throws ResourceHandlerException {
+ Configuration conf, Context nmContext)
+ throws ResourceHandlerException {
ArrayList<ResourceHandler> handlerList = new ArrayList<>();
addHandlerIfNotNull(handlerList, getOutboundBandwidthResourceHandler(conf));
addHandlerIfNotNull(handlerList, getDiskResourceHandler(conf));
addHandlerIfNotNull(handlerList, getMemoryResourceHandler(conf));
addHandlerIfNotNull(handlerList, getCGroupsCpuResourceHandler(conf));
+ addHandlersFromConfiguredResourcePlugins(handlerList, conf, nmContext);
resourceHandlerChain = new ResourceHandlerChain(handlerList);
}
+ private static void addHandlersFromConfiguredResourcePlugins(
+ List<ResourceHandler> handlerList, Configuration conf,
+ Context nmContext) throws ResourceHandlerException {
+ ResourcePluginManager pluginManager = nmContext.getResourcePluginManager();
+ if (pluginManager != null) {
+ Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
+ if (pluginMap != null) {
+ for (ResourcePlugin plugin : pluginMap.values()) {
+ addHandlerIfNotNull(handlerList, plugin
+ .createResourceHandler(nmContext,
+ getInitializedCGroupsHandler(conf),
+ PrivilegedOperationExecutor.getInstance(conf)));
+ }
+ }
+ }
+ }
+
public static ResourceHandlerChain getConfiguredResourceHandlerChain(
- Configuration conf) throws ResourceHandlerException {
+ Configuration conf, Context nmContext) throws ResourceHandlerException {
if (resourceHandlerChain == null) {
synchronized (ResourceHandlerModule.class) {
if (resourceHandlerChain == null) {
- initializeConfiguredResourceHandlerChain(conf);
+ initializeConfiguredResourceHandlerChain(conf, nmContext);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
new file mode 100644
index 0000000..d6bae09
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
+
+/**
+ * Allocate GPU resources according to requirements
+ */
+public class GpuResourceAllocator {
+ final static Log LOG = LogFactory.getLog(GpuResourceAllocator.class);
+
+ private Set<Integer> allowedGpuDevices = new TreeSet<>();
+ private Map<Integer, ContainerId> usedDevices = new TreeMap<>();
+ private Context nmContext;
+
+ public GpuResourceAllocator(Context ctx) {
+ this.nmContext = ctx;
+ }
+
+ /**
+ * Contains allowed and denied devices with minor number.
+ * Denied devices will be useful for cgroups devices module to do blacklisting
+ */
+ static class GpuAllocation {
+ private Set<Integer> allowed = Collections.emptySet();
+ private Set<Integer> denied = Collections.emptySet();
+
+ GpuAllocation(Set<Integer> allowed, Set<Integer> denied) {
+ if (allowed != null) {
+ this.allowed = ImmutableSet.copyOf(allowed);
+ }
+ if (denied != null) {
+ this.denied = ImmutableSet.copyOf(denied);
+ }
+ }
+
+ public Set<Integer> getAllowedGPUs() {
+ return allowed;
+ }
+
+ public Set<Integer> getDeniedGPUs() {
+ return denied;
+ }
+ }
+
+ /**
+ * Add GPU to allowed list
+ * @param minorNumber minor number of the GPU device.
+ */
+ public synchronized void addGpu(int minorNumber) {
+ allowedGpuDevices.add(minorNumber);
+ }
+
+ private String getResourceHandlerExceptionMessage(int numRequestedGpuDevices,
+ ContainerId containerId) {
+ return "Failed to find enough GPUs, requestor=" + containerId
+ + ", #RequestedGPUs=" + numRequestedGpuDevices + ", #availableGpus="
+ + getAvailableGpus();
+ }
+
+ @VisibleForTesting
+ public synchronized int getAvailableGpus() {
+ return allowedGpuDevices.size() - usedDevices.size();
+ }
+
+ public synchronized void recoverAssignedGpus(ContainerId containerId)
+ throws ResourceHandlerException {
+ Container c = nmContext.getContainers().get(containerId);
+ if (null == c) {
+ throw new ResourceHandlerException(
+ "This shouldn't happen, cannot find container with id="
+ + containerId);
+ }
+
+ for (Serializable deviceId : c.getResourceMappings().getAssignedResources(
+ GPU_URI)){
+ if (!(deviceId instanceof String)) {
+ throw new ResourceHandlerException(
+ "Trying to recover device id, however it"
+ + " is not String, this shouldn't happen");
+ }
+
+
+ int devId;
+ try {
+ devId = Integer.parseInt((String)deviceId);
+ } catch (NumberFormatException e) {
+ throw new ResourceHandlerException("Failed to recover device id because"
+ + "it is not a valid integer, devId:" + deviceId);
+ }
+
+ // Make sure it is in allowed GPU device.
+ if (!allowedGpuDevices.contains(devId)) {
+ throw new ResourceHandlerException("Try to recover device id = " + devId
+ + " however it is not in allowed device list:" + StringUtils
+ .join(",", allowedGpuDevices));
+ }
+
+ // Make sure it is not occupied by anybody else
+ if (usedDevices.containsKey(devId)) {
+ throw new ResourceHandlerException("Try to recover device id = " + devId
+ + " however it is already assigned to container=" + usedDevices
+ .get(devId) + ", please double check what happened.");
+ }
+
+ usedDevices.put(devId, containerId);
+ }
+ }
+
+ private int getRequestedGpus(Resource requestedResource) {
+ try {
+ return Long.valueOf(requestedResource.getResourceValue(
+ GPU_URI)).intValue();
+ } catch (ResourceNotFoundException e) {
+ return 0;
+ }
+ }
+
+ /**
+ * Assign GPU to requestor
+ * @param container container to allocate
+ * @return List of denied Gpus with minor numbers
+ * @throws ResourceHandlerException When failed to
+ */
+ public synchronized GpuAllocation assignGpus(Container container)
+ throws ResourceHandlerException {
+ Resource requestedResource = container.getResource();
+ ContainerId containerId = container.getContainerId();
+ int numRequestedGpuDevices = getRequestedGpus(requestedResource);
+ // Assign Gpus to container if requested some.
+ if (numRequestedGpuDevices > 0) {
+ if (numRequestedGpuDevices > getAvailableGpus()) {
+ throw new ResourceHandlerException(
+ getResourceHandlerExceptionMessage(numRequestedGpuDevices,
+ containerId));
+ }
+
+ Set<Integer> assignedGpus = new HashSet<>();
+
+ for (int deviceNum : allowedGpuDevices) {
+ if (!usedDevices.containsKey(deviceNum)) {
+ usedDevices.put(deviceNum, containerId);
+ assignedGpus.add(deviceNum);
+ if (assignedGpus.size() == numRequestedGpuDevices) {
+ break;
+ }
+ }
+ }
+
+ // Record in state store if we allocated anything
+ if (!assignedGpus.isEmpty()) {
+ List<Serializable> allocatedDevices = new ArrayList<>();
+ for (int gpu : assignedGpus) {
+ allocatedDevices.add(String.valueOf(gpu));
+ }
+ try {
+ // Update Container#getResourceMapping.
+ ResourceMappings.AssignedResources assignedResources =
+ new ResourceMappings.AssignedResources();
+ assignedResources.updateAssignedResources(allocatedDevices);
+ container.getResourceMappings().addAssignedResources(GPU_URI,
+ assignedResources);
+
+ // Update state store.
+ nmContext.getNMStateStore().storeAssignedResources(containerId,
+ GPU_URI, allocatedDevices);
+ } catch (IOException e) {
+ cleanupAssignGpus(containerId);
+ throw new ResourceHandlerException(e);
+ }
+ }
+
+ return new GpuAllocation(assignedGpus,
+ Sets.difference(allowedGpuDevices, assignedGpus));
+ }
+ return new GpuAllocation(null, allowedGpuDevices);
+ }
+
+ /**
+ * Clean up all Gpus assigned to containerId
+ * @param containerId containerId
+ */
+ public synchronized void cleanupAssignGpus(ContainerId containerId) {
+ Iterator<Map.Entry<Integer, ContainerId>> iter =
+ usedDevices.entrySet().iterator();
+ while (iter.hasNext()) {
+ if (iter.next().getValue().equals(containerId)) {
+ iter.remove();
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public synchronized Map<Integer, ContainerId> getDeviceAllocationMapping() {
+ return new HashMap<>(usedDevices);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
new file mode 100644
index 0000000..7144bb2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class GpuResourceHandlerImpl implements ResourceHandler {
+ final static Log LOG = LogFactory
+ .getLog(GpuResourceHandlerImpl.class);
+
+ // This will be used by container-executor to add necessary clis
+ public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
+ public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
+
+ private GpuResourceAllocator gpuAllocator;
+ private CGroupsHandler cGroupsHandler;
+ private PrivilegedOperationExecutor privilegedOperationExecutor;
+
+ public GpuResourceHandlerImpl(Context nmContext,
+ CGroupsHandler cGroupsHandler,
+ PrivilegedOperationExecutor privilegedOperationExecutor) {
+ this.cGroupsHandler = cGroupsHandler;
+ this.privilegedOperationExecutor = privilegedOperationExecutor;
+ gpuAllocator = new GpuResourceAllocator(nmContext);
+ }
+
+ @Override
+ public List<PrivilegedOperation> bootstrap(Configuration configuration)
+ throws ResourceHandlerException {
+ List<Integer> minorNumbersOfUsableGpus;
+ try {
+ minorNumbersOfUsableGpus = GpuDiscoverer.getInstance()
+ .getMinorNumbersOfGpusUsableByYarn();
+ } catch (YarnException e) {
+ LOG.error("Exception when trying to get usable GPU device", e);
+ throw new ResourceHandlerException(e);
+ }
+
+ for (int minorNumber : minorNumbersOfUsableGpus) {
+ gpuAllocator.addGpu(minorNumber);
+ }
+
+ // And initialize cgroups
+ this.cGroupsHandler.initializeCGroupController(
+ CGroupsHandler.CGroupController.DEVICES);
+
+ return null;
+ }
+
+ @Override
+ public synchronized List<PrivilegedOperation> preStart(Container container)
+ throws ResourceHandlerException {
+ String containerIdStr = container.getContainerId().toString();
+
+ // Assign Gpus to container if requested some.
+ GpuResourceAllocator.GpuAllocation allocation = gpuAllocator.assignGpus(
+ container);
+
+ // Create device cgroups for the container
+ cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
+ containerIdStr);
+ try {
+ // Execute c-e to setup GPU isolation before launch the container
+ PrivilegedOperation privilegedOperation = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.GPU, Arrays
+ .asList(CONTAINER_ID_CLI_OPTION, containerIdStr));
+ if (!allocation.getDeniedGPUs().isEmpty()) {
+ privilegedOperation.appendArgs(Arrays.asList(EXCLUDED_GPUS_CLI_OPTION,
+ StringUtils.join(",", allocation.getDeniedGPUs())));
+ }
+
+ privilegedOperationExecutor.executePrivilegedOperation(
+ privilegedOperation, true);
+ } catch (PrivilegedOperationException e) {
+ cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.DEVICES,
+ containerIdStr);
+ LOG.warn("Could not update cgroup for container", e);
+ throw new ResourceHandlerException(e);
+ }
+
+ List<PrivilegedOperation> ret = new ArrayList<>();
+ ret.add(new PrivilegedOperation(
+ PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP,
+ PrivilegedOperation.CGROUP_ARG_PREFIX
+ + cGroupsHandler.getPathForCGroupTasks(
+ CGroupsHandler.CGroupController.DEVICES, containerIdStr)));
+
+ return ret;
+ }
+
+ @VisibleForTesting
+ public GpuResourceAllocator getGpuAllocator() {
+ return gpuAllocator;
+ }
+
+ @Override
+ public List<PrivilegedOperation> reacquireContainer(ContainerId containerId)
+ throws ResourceHandlerException {
+ gpuAllocator.recoverAssignedGpus(containerId);
+ return null;
+ }
+
+ @Override
+ public synchronized List<PrivilegedOperation> postComplete(
+ ContainerId containerId) throws ResourceHandlerException {
+ gpuAllocator.cleanupAssignGpus(containerId);
+ cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.DEVICES,
+ containerId.toString());
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/NodeResourceUpdaterPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/NodeResourceUpdaterPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/NodeResourceUpdaterPlugin.java
new file mode 100644
index 0000000..88f77ed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/NodeResourceUpdaterPlugin.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Plugins to handle resources on a node. This will be used by
+ * {@link org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater}
+ */
+public abstract class NodeResourceUpdaterPlugin {
+ /**
+ * Update configured resource for the given component.
+ * @param res resource passed in by external mododule (such as
+ * {@link org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater}
+ * @throws YarnException when any issue happens.
+ */
+ public abstract void updateConfiguredResource(Resource res)
+ throws YarnException;
+
+ /**
+ * This method will be called when the node's resource is loaded from
+ * dynamic-resources.xml in ResourceManager.
+ *
+ * @param newResource newResource reported by RM
+ * @throws YarnException when any mismatch between NM/RM
+ */
+ public void handleUpdatedResourceFromRM(Resource newResource) throws
+ YarnException {
+ // by default do nothing, subclass should implement this method when any
+ // special activities required upon new resource reported by RM.
+ }
+
+ // TODO: add implementation to update node attribute once YARN-3409 merged.
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
new file mode 100644
index 0000000..6e134b3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
+
+/**
+ * {@link ResourcePlugin} is an interface for node manager to easier support
+ * discovery/manage/isolation for new resource types.
+ *
+ * <p>
+ * It has two major part: {@link ResourcePlugin#createResourceHandler(Context,
+ * CGroupsHandler, PrivilegedOperationExecutor)} and
+ * {@link ResourcePlugin#getNodeResourceHandlerInstance()}, see javadocs below
+ * for more details.
+ * </p>
+ */
+public interface ResourcePlugin {
+ /**
+ * Initialize the plugin, this will be invoked during NM startup.
+ * @param context NM Context
+ * @throws YarnException when any issue occurs
+ */
+ void initialize(Context context) throws YarnException;
+
+ /**
+ * Plugin needs to return {@link ResourceHandler} when any special isolation
+ * required for the resource type. This will be added to
+ * {@link ResourceHandlerChain} during NodeManager startup. When no special
+ * isolation need, return null.
+ *
+ * @param nmContext NodeManager context.
+ * @param cGroupsHandler CGroupsHandler
+ * @param privilegedOperationExecutor Privileged Operation Executor.
+ * @return ResourceHandler
+ */
+ ResourceHandler createResourceHandler(Context nmContext,
+ CGroupsHandler cGroupsHandler,
+ PrivilegedOperationExecutor privilegedOperationExecutor);
+
+ /**
+ * Plugin needs to return {@link NodeResourceUpdaterPlugin} when any discovery
+ * mechanism required for the resource type. For example, if we want to set
+ * resource-value during NM registration or send update during NM-RM heartbeat
+ * We can implement a {@link NodeResourceUpdaterPlugin} and update fields of
+ * {@link org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest}
+ * or {@link org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest}
+ *
+ * This will be invoked during every node status update or node registration,
+ * please avoid creating new instance every time.
+ *
+ * @return NodeResourceUpdaterPlugin, could be null when no discovery needed.
+ */
+ NodeResourceUpdaterPlugin getNodeResourceHandlerInstance();
+
+ /**
+ * Do cleanup of the plugin, this will be invoked when
+ * {@link org.apache.hadoop.yarn.server.nodemanager.NodeManager} stops
+ * @throws YarnException if any issue occurs
+ */
+ void cleanup() throws YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
new file mode 100644
index 0000000..73d6038
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
+
+/**
+ * Manages {@link ResourcePlugin} configured on this NodeManager.
+ */
+public class ResourcePluginManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ResourcePluginManager.class);
+ private static final Set<String> SUPPORTED_RESOURCE_PLUGINS = ImmutableSet.of(
+ GPU_URI);
+
+ private Map<String, ResourcePlugin> configuredPlugins = Collections.EMPTY_MAP;
+
+ public synchronized void initialize(Context context)
+ throws YarnException {
+ Configuration conf = context.getConf();
+ String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+
+ if (plugins != null) {
+ Map<String, ResourcePlugin> pluginMap = new HashMap<>();
+
+ // Initialize each plugins
+ for (String resourceName : plugins) {
+ resourceName = resourceName.trim();
+ if (!SUPPORTED_RESOURCE_PLUGINS.contains(resourceName)) {
+ String msg =
+ "Trying to initialize resource plugin with name=" + resourceName
+ + ", it is not supported, list of supported plugins:"
+ + StringUtils.join(",",
+ SUPPORTED_RESOURCE_PLUGINS);
+ LOG.error(msg);
+ throw new YarnException(msg);
+ }
+
+ if (pluginMap.containsKey(resourceName)) {
+ // Duplicated items, ignore ...
+ continue;
+ }
+
+ ResourcePlugin plugin = null;
+ if (resourceName.equals(GPU_URI)) {
+ plugin = new GpuResourcePlugin();
+ }
+
+ if (plugin == null) {
+ throw new YarnException(
+ "This shouldn't happen, plugin=" + resourceName
+ + " should be loaded and initialized");
+ }
+ plugin.initialize(context);
+ pluginMap.put(resourceName, plugin);
+ }
+
+ configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ }
+ }
+
+ public synchronized void cleanup() throws YarnException {
+ for (ResourcePlugin plugin : configuredPlugins.values()) {
+ plugin.cleanup();
+ }
+ }
+
+ /**
+ * Get resource name (such as gpu/fpga) to plugin references.
+ * @return read-only map of resource name to plugins.
+ */
+ public synchronized Map<String, ResourcePlugin> getNameToPlugins() {
+ return configuredPlugins;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
new file mode 100644
index 0000000..61b8ce5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformationParser;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.PerGpuDeviceInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class GpuDiscoverer {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ GpuDiscoverer.class);
+ @VisibleForTesting
+ protected static final String DEFAULT_BINARY_NAME = "nvidia-smi";
+
+ // When executable path not set, try to search default dirs
+ // By default search /usr/bin, /bin, and /usr/local/nvidia/bin (when
+ // launched by nvidia-docker.
+ private static final Set<String> DEFAULT_BINARY_SEARCH_DIRS = ImmutableSet.of(
+ "/usr/bin", "/bin", "/usr/local/nvidia/bin");
+
+ // command should not run more than 10 sec.
+ private static final int MAX_EXEC_TIMEOUT_MS = 10 * 1000;
+ private static final int MAX_REPEATED_ERROR_ALLOWED = 10;
+ private static GpuDiscoverer instance;
+
+ static {
+ instance = new GpuDiscoverer();
+ }
+
+ private Configuration conf = null;
+ private String pathOfGpuBinary = null;
+ private Map<String, String> environment = new HashMap<>();
+ private GpuDeviceInformationParser parser = new GpuDeviceInformationParser();
+
+ private int numOfErrorExecutionSinceLastSucceed = 0;
+ GpuDeviceInformation lastDiscoveredGpuInformation = null;
+
+ private void validateConfOrThrowException() throws YarnException {
+ if (conf == null) {
+ throw new YarnException("Please initialize (call initialize) before use "
+ + GpuDiscoverer.class.getSimpleName());
+ }
+ }
+
+ /**
+ * Get GPU device information from system.
+ * This need to be called after initialize.
+ *
+ * Please note that this only works on *NIX platform, so external caller
+ * need to make sure this.
+ *
+ * @return GpuDeviceInformation
+ * @throws YarnException when any error happens
+ */
+ public synchronized GpuDeviceInformation getGpuDeviceInformation()
+ throws YarnException {
+ validateConfOrThrowException();
+
+ if (null == pathOfGpuBinary) {
+ throw new YarnException(
+ "Failed to find GPU discovery executable, please double check "
+ + YarnConfiguration.NM_GPU_PATH_TO_EXEC + " setting.");
+ }
+
+ if (numOfErrorExecutionSinceLastSucceed == MAX_REPEATED_ERROR_ALLOWED) {
+ String msg =
+ "Failed to execute GPU device information detection script for "
+ + MAX_REPEATED_ERROR_ALLOWED
+ + " times, skip following executions.";
+ LOG.error(msg);
+ throw new YarnException(msg);
+ }
+
+ String output;
+ try {
+ output = Shell.execCommand(environment,
+ new String[] { pathOfGpuBinary, "-x", "-q" }, MAX_EXEC_TIMEOUT_MS);
+ GpuDeviceInformation info = parser.parseXml(output);
+ numOfErrorExecutionSinceLastSucceed = 0;
+ lastDiscoveredGpuInformation = info;
+ return info;
+ } catch (IOException e) {
+ numOfErrorExecutionSinceLastSucceed++;
+ String msg =
+ "Failed to execute " + pathOfGpuBinary + " exception message:" + e
+ .getMessage() + ", continue ...";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(msg);
+ }
+ throw new YarnException(e);
+ } catch (YarnException e) {
+ numOfErrorExecutionSinceLastSucceed++;
+ String msg = "Failed to parse xml output" + e.getMessage();
+ if (LOG.isDebugEnabled()) {
+ LOG.warn(msg, e);
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Get list of minor device numbers of Gpu devices usable by YARN.
+ *
+ * @return List of minor device numbers of Gpu devices.
+ * @throws YarnException when any issue happens
+ */
+ public synchronized List<Integer> getMinorNumbersOfGpusUsableByYarn()
+ throws YarnException {
+ validateConfOrThrowException();
+
+ String allowedDevicesStr = conf.get(
+ YarnConfiguration.NM_GPU_ALLOWED_DEVICES,
+ YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES);
+
+ List<Integer> minorNumbers = new ArrayList<>();
+
+ if (allowedDevicesStr.equals(
+ YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES)) {
+ // Get gpu device information from system.
+ if (null == lastDiscoveredGpuInformation) {
+ String msg = YarnConfiguration.NM_GPU_ALLOWED_DEVICES + " is set to "
+ + YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES
+ + ", however automatically discovering "
+ + "GPU information failed, please check NodeManager log for more"
+ + " details, as an alternative, admin can specify "
+ + YarnConfiguration.NM_GPU_ALLOWED_DEVICES
+ + " manually to enable GPU isolation.";
+ LOG.error(msg);
+ throw new YarnException(msg);
+ }
+
+ if (lastDiscoveredGpuInformation.getGpus() != null) {
+ for (PerGpuDeviceInformation gpu : lastDiscoveredGpuInformation
+ .getGpus()) {
+ minorNumbers.add(gpu.getMinorNumber());
+ }
+ }
+ } else{
+ for (String s : allowedDevicesStr.split(",")) {
+ if (s.trim().length() > 0) {
+ minorNumbers.add(Integer.valueOf(s.trim()));
+ }
+ }
+ LOG.info("Allowed GPU devices with minor numbers:" + allowedDevicesStr);
+ }
+
+ return minorNumbers;
+ }
+
+ public synchronized void initialize(Configuration conf) throws YarnException {
+ this.conf = conf;
+ numOfErrorExecutionSinceLastSucceed = 0;
+ String pathToExecutable = conf.get(YarnConfiguration.NM_GPU_PATH_TO_EXEC,
+ YarnConfiguration.DEFAULT_NM_GPU_PATH_TO_EXEC);
+ if (pathToExecutable.isEmpty()) {
+ pathToExecutable = DEFAULT_BINARY_NAME;
+ }
+
+ // Validate file existence
+ File binaryPath = new File(pathToExecutable);
+
+ if (!binaryPath.exists()) {
+ // When binary not exist, use default setting.
+ boolean found = false;
+ for (String dir : DEFAULT_BINARY_SEARCH_DIRS) {
+ binaryPath = new File(dir, DEFAULT_BINARY_NAME);
+ if (binaryPath.exists()) {
+ found = true;
+ pathOfGpuBinary = binaryPath.getAbsolutePath();
+ break;
+ }
+ }
+
+ if (!found) {
+ LOG.warn("Failed to locate binary at:" + binaryPath.getAbsolutePath()
+ + ", please double check [" + YarnConfiguration.NM_GPU_PATH_TO_EXEC
+ + "] setting. Now use " + "default binary:" + DEFAULT_BINARY_NAME);
+ }
+ } else{
+ // If path specified by user is a directory, use
+ if (binaryPath.isDirectory()) {
+ binaryPath = new File(binaryPath, DEFAULT_BINARY_NAME);
+ LOG.warn("Specified path is a directory, use " + DEFAULT_BINARY_NAME
+ + " under the directory, updated path-to-executable:" + binaryPath
+ .getAbsolutePath());
+ }
+ // Validated
+ pathOfGpuBinary = binaryPath.getAbsolutePath();
+ }
+
+ // Try to discover GPU information once and print
+ try {
+ LOG.info("Trying to discover GPU information ...");
+ GpuDeviceInformation info = getGpuDeviceInformation();
+ LOG.info(info.toString());
+ } catch (YarnException e) {
+ String msg =
+ "Failed to discover GPU information from system, exception message:"
+ + e.getMessage() + " continue...";
+ LOG.warn(msg);
+ }
+ }
+
+ @VisibleForTesting
+ protected Map<String, String> getEnvironmentToRunCommand() {
+ return environment;
+ }
+
+ @VisibleForTesting
+ protected String getPathOfGpuBinary() {
+ return pathOfGpuBinary;
+ }
+
+ public static GpuDiscoverer getInstance() {
+ return instance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa5cfc68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java
new file mode 100644
index 0000000..f6bf506
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
+
+public class GpuNodeResourceUpdateHandler extends NodeResourceUpdaterPlugin {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GpuNodeResourceUpdateHandler.class);
+
+ @Override
+ public void updateConfiguredResource(Resource res) throws YarnException {
+ LOG.info("Initializing configured GPU resources for the NodeManager.");
+
+ List<Integer> usableGpus =
+ GpuDiscoverer.getInstance().getMinorNumbersOfGpusUsableByYarn();
+ if (null == usableGpus || usableGpus.isEmpty()) {
+ LOG.info("Didn't find any usable GPUs on the NodeManager.");
+ // No gpu can be used by YARN.
+ return;
+ }
+
+ long nUsableGpus = usableGpus.size();
+
+ Map<String, ResourceInformation> configuredResourceTypes =
+ ResourceUtils.getResourceTypes();
+ if (!configuredResourceTypes.containsKey(GPU_URI)) {
+ throw new YarnException("Found " + nUsableGpus + " usable GPUs, however "
+ + GPU_URI
+ + " resource-type is not configured inside"
+ + " resource-types.xml, please configure it to enable GPU feature or"
+ + " remove " + GPU_URI + " from "
+ + YarnConfiguration.NM_RESOURCE_PLUGINS);
+ }
+
+ res.setResourceValue(GPU_URI, nUsableGpus);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-6840. Implement zookeeper
based store for scheduler configuration updates. (Jonathan Hung via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
new file mode 100644
index 0000000..3cfa8da
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -0,0 +1,312 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Tests {@link ZKConfigurationStore}.
+ */
+public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestZKConfigurationStore.class);
+
+ private static final int ZK_TIMEOUT_MS = 10000;
+ private TestingServer curatorTestingServer;
+ private CuratorFramework curatorFramework;
+ private ResourceManager rm;
+
+ public static TestingServer setupCuratorServer() throws Exception {
+ TestingServer curatorTestingServer = new TestingServer();
+ curatorTestingServer.start();
+ return curatorTestingServer;
+ }
+
+ public static CuratorFramework setupCuratorFramework(
+ TestingServer curatorTestingServer) throws Exception {
+ CuratorFramework curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(curatorTestingServer.getConnectString())
+ .retryPolicy(new RetryNTimes(100, 100))
+ .build();
+ curatorFramework.start();
+ return curatorFramework;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ curatorTestingServer = setupCuratorServer();
+ curatorFramework = setupCuratorFramework(curatorTestingServer);
+
+ conf.set(CommonConfigurationKeys.ZK_ADDRESS,
+ curatorTestingServer.getConnectString());
+ rm = new MockRM(conf);
+ rm.start();
+ rmContext = rm.getRMContext();
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ rm.stop();
+ curatorFramework.close();
+ curatorTestingServer.stop();
+ }
+
+ @Test
+ public void testVersioning() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ assertEquals(ZKConfigurationStore.CURRENT_VERSION_INFO,
+ confStore.getConfStoreVersion());
+ }
+
+ @Test
+ public void testPersistConfiguration() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+
+ // Create a new configuration store, and check for old configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ }
+
+
+ @Test
+ public void testPersistUpdatedConfiguration() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.retrieve().get("key"));
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key", "val");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+ confStore.confirmMutation(true);
+ assertEquals("val", confStore.retrieve().get("key"));
+
+ // Create a new configuration store, and check for updated configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ }
+
+ @Test
+ public void testMaxLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
+ confStore.initialize(conf, schedConf, rmContext);
+ LinkedList<YarnConfigurationStore.LogMutation> logs =
+ ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(0, logs.size());
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("key1", "val1");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update1, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ confStore.confirmMutation(true);
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("key2", "val2");
+ mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+
+ // Next update should purge first update from logs.
+ Map<String, String> update3 = new HashMap<>();
+ update3.put("key3", "val3");
+ mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ }
+
+ public Configuration createRMHAConf(String rmIds, String rmId,
+ int adminPort) {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.ZK_CONFIGURATION_STORE);
+ conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS,
+ curatorTestingServer.getConnectString());
+ conf.set(YarnConfiguration.RM_HA_ID, rmId);
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ for (String rpcAddress :
+ YarnConfiguration.getServiceAddressConfKeys(conf)) {
+ for (String id : HAUtil.getRMHAIds(conf)) {
+ conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0");
+ }
+ }
+ conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId),
+ "localhost:" + adminPort);
+ return conf;
+ }
+
+ /**
+ * When failing over, new active RM should read from current state of store,
+ * including any updates when the new active RM was in standby.
+ * @throws Exception
+ */
+ @Test
+ public void testFailoverReadsFromUpdatedStore() throws Exception {
+ HAServiceProtocol.StateChangeRequestInfo req =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+
+ Configuration conf1 = createRMHAConf("rm1,rm2", "rm1", 1234);
+ ResourceManager rm1 = new MockRM(conf1);
+ rm1.start();
+ rm1.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm1.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+ assertNull(((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+
+ Configuration conf2 = createRMHAConf("rm1,rm2", "rm2", 5678);
+ ResourceManager rm2 = new MockRM(conf2);
+ rm2.start();
+ assertEquals("RM should be Standby",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ // Update configuration on RM1
+ SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo();
+ schedConfUpdateInfo.getGlobalParams().put("key", "val");
+ MutableConfigurationProvider confProvider = ((MutableConfScheduler)
+ rm1.getResourceScheduler()).getMutableConfProvider();
+ UserGroupInformation user = UserGroupInformation
+ .createUserForTesting(TEST_USER, new String[0]);
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext());
+ assertEquals("val", ((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+ confProvider.confirmPendingMutation(true);
+ assertEquals("val", ((MutableCSConfigurationProvider) confProvider)
+ .getConfStore().retrieve().get("key"));
+ // Next update is not persisted, it should not be recovered
+ schedConfUpdateInfo.getGlobalParams().put("key", "badVal");
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+
+ // Start RM2 and verifies it starts with updated configuration
+ rm2.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm2.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
+ if (HAServiceProtocol.HAServiceState.ACTIVE ==
+ rm1.getRMContext().getRMAdminService().getServiceStatus()
+ .getState()) {
+ Thread.sleep(100);
+ }
+ }
+ assertEquals("RM should have been fenced",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ assertEquals("val", ((MutableCSConfigurationProvider) (
+ (CapacityScheduler) rm2.getResourceScheduler())
+ .getMutableConfProvider()).getConfStore().retrieve().get("key"));
+ assertEquals("val", ((MutableConfScheduler) rm2.getResourceScheduler())
+ .getConfiguration().get("key"));
+ // Transition to standby will set RM's HA status and then reinitialize in
+ // a separate thread. Despite asserting for STANDBY state, it's
+ // possible for reinitialization to be unfinished. Wait here for it to
+ // finish, otherwise closing rm1 will close zkManager and the unfinished
+ // reinitialization will throw an exception.
+ Thread.sleep(10000);
+ rm1.close();
+ rm2.close();
+ }
+
+ @Override
+ public YarnConfigurationStore createConfStore() {
+ return new ZKConfigurationStore();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-7252. Removing queue then
failing over results in exception
Posted by su...@apache.org.
YARN-7252. Removing queue then failing over results in exception
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d36b53a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d36b53a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d36b53a
Branch: refs/heads/YARN-5881
Commit: 1d36b53ab6d9bb1d9144101e424c24371343c5bf
Parents: 09c5dfe
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Tue Sep 26 11:41:05 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../capacity/CapacitySchedulerContext.java | 6 ++
.../capacity/CapacitySchedulerQueueManager.java | 11 +-
.../capacity/conf/TestZKConfigurationStore.java | 102 +++++++++++++++++++
3 files changed, 117 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d36b53a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 9aeaec6..7c918a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -88,4 +88,10 @@ public interface CapacitySchedulerContext {
* @return Max Cluster level App priority.
*/
Priority getMaxClusterLevelAppPriority();
+
+ /**
+ * Returns if configuration is mutable.
+ * @return if configuration is mutable
+ */
+ boolean isConfigurationMutable();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d36b53a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 1ceb6fb..48c289f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueState;
@@ -170,8 +171,14 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
CSQueue newRoot = parseQueue(this.csContext, newConf, null,
CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP);
- // Ensure queue hiearchy in the new XML file is proper.
- validateQueueHierarchy(queues, newQueues);
+ // When failing over, if using configuration store, don't validate queue
+ // hierarchy since queues can be removed without being STOPPED.
+ if (!csContext.isConfigurationMutable() ||
+ csContext.getRMContext().getHAServiceState()
+ != HAServiceProtocol.HAServiceState.STANDBY) {
+ // Ensure queue hiearchy in the new XML file is proper.
+ validateQueueHierarchy(queues, newQueues);
+ }
// Add new queues and delete OldQeueus only after validation.
updateQueues(queues, newQueues);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d36b53a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 355f741..0cf5e6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -38,18 +38,21 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfSchedu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
/**
* Tests {@link ZKConfigurationStore}.
@@ -303,6 +306,105 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
rm2.close();
}
+ /**
+ * When failing over, if RM1 stopped and removed a queue that RM2 has in
+ * memory, failing over to RM2 should not throw an exception.
+ * @throws Exception
+ */
+ @Test
+ public void testFailoverAfterRemoveQueue() throws Exception {
+ HAServiceProtocol.StateChangeRequestInfo req =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+
+ Configuration conf1 = createRMHAConf("rm1,rm2", "rm1", 1234);
+ ResourceManager rm1 = new MockRM(conf1);
+ rm1.start();
+ rm1.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm1.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ Configuration conf2 = createRMHAConf("rm1,rm2", "rm2", 5678);
+ ResourceManager rm2 = new MockRM(conf2);
+ rm2.start();
+ assertEquals("RM should be Standby",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ UserGroupInformation user = UserGroupInformation
+ .createUserForTesting(TEST_USER, new String[0]);
+ MutableConfigurationProvider confProvider = ((MutableConfScheduler)
+ rm1.getResourceScheduler()).getMutableConfProvider();
+ // Add root.a
+ SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo();
+ Map<String, String> addParams = new HashMap<>();
+ addParams.put("capacity", "100");
+ QueueConfigInfo addInfo = new QueueConfigInfo("root.a", addParams);
+ schedConfUpdateInfo.getAddQueueInfo().add(addInfo);
+ // Stop root.default
+ Map<String, String> stopParams = new HashMap<>();
+ stopParams.put("state", "STOPPED");
+ stopParams.put("capacity", "0");
+ QueueConfigInfo stopInfo = new QueueConfigInfo("root.default", stopParams);
+ schedConfUpdateInfo.getUpdateQueueInfo().add(stopInfo);
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext());
+ confProvider.confirmPendingMutation(true);
+ assertTrue(Arrays.asList(((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("yarn.scheduler.capacity.root.queues").split
+ (",")).contains("a"));
+
+ // Remove root.default
+ schedConfUpdateInfo.getUpdateQueueInfo().clear();
+ schedConfUpdateInfo.getAddQueueInfo().clear();
+ schedConfUpdateInfo.getRemoveQueueInfo().add("root.default");
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext());
+ confProvider.confirmPendingMutation(true);
+ assertEquals("a", ((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("yarn.scheduler.capacity.root.queues"));
+
+ // Start RM2 and verifies it starts with updated configuration
+ rm2.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm2.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
+ if (HAServiceProtocol.HAServiceState.ACTIVE ==
+ rm1.getRMContext().getRMAdminService().getServiceStatus()
+ .getState()) {
+ Thread.sleep(100);
+ }
+ }
+ assertEquals("RM should have been fenced",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ assertEquals("a", ((MutableCSConfigurationProvider) (
+ (CapacityScheduler) rm2.getResourceScheduler())
+ .getMutableConfProvider()).getConfStore().retrieve()
+ .get("yarn.scheduler.capacity.root.queues"));
+ assertEquals("a", ((MutableConfScheduler) rm2.getResourceScheduler())
+ .getConfiguration().get("yarn.scheduler.capacity.root.queues"));
+ // Transition to standby will set RM's HA status and then reinitialize in
+ // a separate thread. Despite asserting for STANDBY state, it's
+ // possible for reinitialization to be unfinished. Wait here for it to
+ // finish, otherwise closing rm1 will close zkManager and the unfinished
+ // reinitialization will throw an exception.
+ Thread.sleep(10000);
+ rm1.close();
+ rm2.close();
+ }
+
@Override
public YarnConfigurationStore createConfStore() {
return new ZKConfigurationStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-5952. Create REST API for
changing YARN scheduler configurations. (Jonathan Hung via wangda)
Posted by su...@apache.org.
YARN-5952. Create REST API for changing YARN scheduler configurations. (Jonathan Hung via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8a5136b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8a5136b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8a5136b
Branch: refs/heads/YARN-5881
Commit: d8a5136b1f3761166a08969eda7a7f66d1578d7e
Parents: ef68663
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Apr 3 10:12:01 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../scheduler/MutableConfScheduler.java | 40 ++
.../scheduler/MutableConfigurationProvider.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../conf/InMemoryConfigurationStore.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 24 +-
.../resourcemanager/webapp/RMWebServices.java | 172 ++++++-
.../webapp/dao/QueueConfigInfo.java | 57 +++
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 +++
.../TestMutableCSConfigurationProvider.java | 6 +-
.../TestRMWebServicesConfigurationMutation.java | 477 +++++++++++++++++++
10 files changed, 851 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
new file mode 100644
index 0000000..35e36e1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface for a scheduler that supports changing configuration at runtime.
+ *
+ */
+public interface MutableConfScheduler extends ResourceScheduler {
+
+ /**
+ * Update the scheduler's configuration.
+ * @param user Caller of this update
+ * @param confUpdate key-value map of the configuration update
+ * @throws IOException if update is invalid
+ */
+ void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index da30a2b..889c3bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.io.IOException;
import java.util.Map;
/**
@@ -29,7 +30,9 @@ public interface MutableConfigurationProvider {
* Update the scheduler configuration with the provided key value pairs.
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
+ * @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate);
+ void mutateConfiguration(String user, Map<String, String> confUpdate)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 11f2f6e..7c5839b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -86,6 +86,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -150,7 +152,7 @@ import com.google.common.util.concurrent.SettableFuture;
public class CapacityScheduler extends
AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> implements
PreemptableResourceScheduler, CapacitySchedulerContext, Configurable,
- ResourceAllocationCommitter {
+ ResourceAllocationCommitter, MutableConfScheduler {
private static final Log LOG = LogFactory.getLog(CapacityScheduler.class);
@@ -2610,4 +2612,16 @@ public class CapacityScheduler extends
// In seconds
return ((LeafQueue) queue).getMaximumApplicationLifetime();
}
+
+ @Override
+ public void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException {
+ if (csConfProvider instanceof MutableConfigurationProvider) {
+ ((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
+ user.getShortUserName(), confUpdate);
+ } else {
+ throw new UnsupportedOperationException("Configured CS configuration " +
+ "provider does not support updating configuration.");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index a208fb9..b97be1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -58,7 +58,11 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
if (isValid) {
Map<String, String> mutations = mutation.getUpdates();
for (Map.Entry<String, String> kv : mutations.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 267ab6a..ea1b3c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -60,34 +60,44 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
- this.schedConf = initialSchedConf;
- confStore.initialize(config, initialSchedConf);
+ this.schedConf = new Configuration(false);
+ // We need to explicitly set the key-values in schedConf, otherwise
+ // these configuration keys cannot be deleted when
+ // configuration is reloaded.
+ for (Map.Entry<String, String> kv : initialSchedConf) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ confStore.initialize(config, schedConf);
this.conf = config;
}
@Override
public CapacitySchedulerConfiguration loadConfiguration(Configuration
configuration) throws IOException {
- Configuration loadedConf = new Configuration(configuration);
- loadedConf.addResource(schedConf);
+ Configuration loadedConf = new Configuration(schedConf);
+ loadedConf.addResource(configuration);
return new CapacitySchedulerConfiguration(loadedConf, false);
}
@Override
public void mutateConfiguration(String user,
- Map<String, String> confUpdate) {
+ Map<String, String> confUpdate) throws IOException {
Configuration oldConf = new Configuration(schedConf);
LogMutation log = new LogMutation(confUpdate, user);
long id = confStore.logMutation(log);
for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
try {
rmContext.getScheduler().reinitialize(conf, rmContext);
} catch (IOException e) {
schedConf = oldConf;
confStore.confirmMutation(id, false);
- return;
+ throw e;
}
confStore.confirmMutation(id, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6dc3d9a..215e511 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -55,7 +55,8 @@ import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
-
+import com.google.common.base.Joiner;
+import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -134,11 +135,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2454,4 +2458,170 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
GetContainersRequest request) throws YarnException, IOException {
return rm.getClientRMService().getContainers(request).getContainerList();
}
+
+ @PUT
+ @Path("/queues")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ mutationInfo, @Context HttpServletRequest hsr)
+ throws AuthorizationException, InterruptedException {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ ApplicationACLsManager aclsManager = rm.getApplicationACLsManager();
+ if (aclsManager.areACLsEnabled()) {
+ if (callerUGI == null || !aclsManager.isAdmin(callerUGI)) {
+ String msg = "Only admins can carry out this operation.";
+ throw new ForbiddenException(msg);
+ }
+ }
+
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ if (scheduler instanceof MutableConfScheduler) {
+ try {
+ callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, YarnException {
+ Map<String, String> confUpdate =
+ constructKeyValueConfUpdate(mutationInfo);
+ ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
+ confUpdate);
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
+ .build();
+ }
+ return Response.status(Status.OK).entity("Configuration change " +
+ "successfully applied.").build();
+ } else {
+ return Response.status(Status.BAD_REQUEST)
+ .entity("Configuration change only supported by CapacityScheduler.")
+ .build();
+ }
+ }
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacitySchedulerConfiguration currentConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(currentConf, false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
+ parentQueuePath + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
+ parentQueue + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
new file mode 100644
index 0000000..b20eda6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for adding or updating a queue to scheduler configuration
+ * for this queue.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigInfo {
+
+ @XmlElement(name = "queueName")
+ private String queue;
+
+ private HashMap<String, String> params = new HashMap<>();
+
+ public QueueConfigInfo() { }
+
+ public QueueConfigInfo(String queue, Map<String, String> params) {
+ this.queue = queue;
+ this.params = new HashMap<>(params);
+ }
+
+ public String getQueue() {
+ return this.queue;
+ }
+
+ public HashMap<String, String> getParams() {
+ return this.params;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
new file mode 100644
index 0000000..644ec90
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigsUpdateInfo {
+
+ @XmlElement(name = "add")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ public QueueConfigsUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 3f103b1..254da31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -77,7 +77,11 @@ public class TestMutableCSConfigurationProvider {
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
- confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ try {
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ } catch (IOException e) {
+ // Expected exception.
+ }
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8a5136b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
new file mode 100644
index 0000000..d149055
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Test scheduler configuration mutation via REST API.
+ */
+public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
+
+ private static final File CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
+ private static final File OLD_CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp");
+
+ private static MockRM rm;
+ private static String userName;
+ private static CapacitySchedulerConfiguration csConf;
+ private static YarnConfiguration conf;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try {
+ userName = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Unable to get current user name "
+ + ioe.getMessage(), ioe);
+ }
+ csConf = new CapacitySchedulerConfiguration(new Configuration(false),
+ false);
+ setupQueueConfiguration(csConf);
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+ try {
+ if (CONF_FILE.exists()) {
+ if (!CONF_FILE.renameTo(OLD_CONF_FILE)) {
+ throw new RuntimeException("Failed to rename conf file");
+ }
+ }
+ FileOutputStream out = new FileOutputStream(CONF_FILE);
+ csConf.writeXml(out);
+ out.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write XML file", e);
+ }
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ filter("/*").through(TestRMWebServicesAppsModification
+ .TestRMCustomAuthFilter.class);
+ }
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ }
+
+ private static void setupQueueConfiguration(
+ CapacitySchedulerConfiguration config) {
+ config.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{"a", "b", "c"});
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ config.setCapacity(a, 25f);
+ config.setMaximumCapacity(a, 50f);
+
+ final String a1 = a + ".a1";
+ final String a2 = a + ".a2";
+ config.setQueues(a, new String[]{"a1", "a2"});
+ config.setCapacity(a1, 100f);
+ config.setCapacity(a2, 0f);
+
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ config.setCapacity(b, 75f);
+
+ final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+ config.setCapacity(c, 0f);
+
+ final String c1 = c + ".c1";
+ config.setQueues(c, new String[] {"c1"});
+ config.setCapacity(c1, 0f);
+ }
+
+ public TestRMWebServicesConfigurationMutation() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testAddNestedQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add parent queue root.d with two children d1 and d2.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> d1Capacity = new HashMap<>();
+ d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
+ Map<String, String> nearEmptyCapacity = new HashMap<>();
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4");
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY,
+ "1E-4");
+ Map<String, String> d2Capacity = new HashMap<>();
+ d2Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "75");
+ d2Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "75");
+ QueueConfigInfo d1 = new QueueConfigInfo("root.d.d1", d1Capacity);
+ QueueConfigInfo d2 = new QueueConfigInfo("root.d.d2", d2Capacity);
+ QueueConfigInfo d = new QueueConfigInfo("root.d", nearEmptyCapacity);
+ updateInfo.getAddQueueInfo().add(d1);
+ updateInfo.getAddQueueInfo().add(d2);
+ updateInfo.getAddQueueInfo().add(d);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(2, newCSConf.getQueues("root.d").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"),
+ 0.01f);
+ assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
+ 0.01f);
+ }
+
+ @Test
+ public void testAddWithUpdate() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> dCapacity = new HashMap<>();
+ dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo d = new QueueConfigInfo("root.d", dCapacity);
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getAddQueueInfo().add(d);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Test
+ public void testRemoveQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a.a2");
+ // Remove root.a.a2
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a.a2");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root.a").length);
+ assertEquals("a1", newCSConf.getQueues("root.a")[0]);
+ }
+
+ @Test
+ public void testRemoveParentQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.c", "root.c.c1");
+ // Remove root.c (parent queue)
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertNull(newCSConf.getQueues("root.c"));
+ }
+
+ @Test
+ public void testRemoveParentQueueWithCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a", "root.a.a1", "root.a.a2");
+ // Remove root.a (parent queue) with capacity 25
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a");
+
+ // Set root.b capacity to 100
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"),
+ 0.01f);
+ }
+
+ @Test
+ public void testRemoveMultipleQueues() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.b", "root.c", "root.c.c1");
+ // Remove root.b and root.c
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b");
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ Map<String, String> aCapacity = new HashMap<>();
+ aCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ aCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "100");
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", aCapacity);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root").length);
+ }
+
+ private void stopQueue(String... queuePaths) throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set state of queues to STOPPED.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> stoppedParam = new HashMap<>();
+ stoppedParam.put(CapacitySchedulerConfiguration.STATE,
+ QueueState.STOPPED.toString());
+ for (String queue : queuePaths) {
+ QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue, stoppedParam);
+ updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+ }
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ for (String queue : queuePaths) {
+ assertEquals(QueueState.STOPPED, newCSConf.getState(queue));
+ }
+ }
+
+ @Test
+ public void testUpdateQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update config value.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ "0.2");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+ cs.getConfiguration()
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
+ assertEquals(0.2f, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), 0.001f);
+
+ // Remove config. Config value should be reverted to default.
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ null);
+ aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().clear();
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf = cs.getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ }
+
+ @Test
+ public void testUpdateQueueCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update root.a and root.b capacity to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ QueueConfigInfo bUpdateInfo = new QueueConfigInfo("root.b", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ updateInfo.getUpdateQueueInfo().add(bUpdateInfo);
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ if (rm != null) {
+ rm.stop();
+ }
+ CONF_FILE.delete();
+ if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
+ throw new RuntimeException("Failed to re-copy old configuration file");
+ }
+ super.tearDown();
+ }
+
+ @SuppressWarnings("rawtypes")
+ private String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-5953:Create CLI for changing
YARN configurations. (Jonathan Hung via xgong)
Posted by su...@apache.org.
YARN-5953:Create CLI for changing YARN configurations. (Jonathan Hung via xgong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/916bdbd6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/916bdbd6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/916bdbd6
Branch: refs/heads/YARN-5881
Commit: 916bdbd6bedbc75000bb74e16c1e2ee692a4912c
Parents: e566fd8
Author: Xuan <xg...@apache.org>
Authored: Fri Jul 7 14:16:46 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 5 +
.../hadoop/yarn/client/cli/SchedConfCLI.java | 238 +++++++++++++++++++
.../yarn/client/cli/TestSchedConfCLI.java | 160 +++++++++++++
.../hadoop/yarn/webapp/dao/package-info.java | 27 +++
.../yarn/webapp/util/YarnWebServiceUtils.java | 14 ++
.../ConfigurationMutationACLPolicy.java | 2 +-
.../DefaultConfigurationMutationACLPolicy.java | 2 +-
.../scheduler/MutableConfScheduler.java | 2 +-
.../scheduler/MutableConfigurationProvider.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../conf/MutableCSConfigurationProvider.java | 4 +-
...ueueAdminConfigurationMutationACLPolicy.java | 4 +-
.../resourcemanager/webapp/RMWebServices.java | 1 +
.../webapp/dao/QueueConfigInfo.java | 4 +-
.../webapp/dao/SchedConfUpdateInfo.java | 18 +-
.../TestConfigurationMutationACLPolicies.java | 4 +-
.../TestMutableCSConfigurationProvider.java | 4 +-
.../TestRMWebServicesConfigurationMutation.java | 65 +++--
19 files changed, 507 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 52138c5..0063596 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -142,6 +142,9 @@ function yarncmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
;;
+ schedconf)
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
+ ;;
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 690badf..7ec9848 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -295,6 +295,11 @@ goto :eof
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
+:schedconf
+ set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
:make_command_arguments
if "%1" == "--config" (
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
new file mode 100644
index 0000000..e17062e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -0,0 +1,238 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.client.cli;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * CLI for modifying scheduler configuration.
+ */
+@Public
+@Evolving
+public class SchedConfCLI extends Configured implements Tool {
+
+ private static final String ADD_QUEUES_OPTION = "addQueues";
+ private static final String REMOVE_QUEUES_OPTION = "removeQueues";
+ private static final String UPDATE_QUEUES_OPTION = "updateQueues";
+ private static final String GLOBAL_OPTIONS = "globalUpdates";
+ private static final String HELP_CMD = "help";
+
+ private static final String CONF_ERR_MSG = "Specify configuration key " +
+ "value as confKey=confVal.";
+
+ public SchedConfCLI() {
+ super(new YarnConfiguration());
+ }
+
+ public static void main(String[] args) throws Exception {
+ SchedConfCLI cli = new SchedConfCLI();
+ int exitCode = cli.run(args);
+ System.exit(exitCode);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options opts = new Options();
+
+ opts.addOption("add", ADD_QUEUES_OPTION, true,
+ "Add queues with configurations");
+ opts.addOption("remove", REMOVE_QUEUES_OPTION, true,
+ "Remove queues");
+ opts.addOption("update", UPDATE_QUEUES_OPTION, true,
+ "Update queue configurations");
+ opts.addOption("global", GLOBAL_OPTIONS, true,
+ "Update global scheduler configurations");
+ opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
+
+ int exitCode = -1;
+ CommandLine parsedCli = null;
+ try {
+ parsedCli = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ System.err.println("Missing argument for options");
+ printUsage();
+ return exitCode;
+ }
+
+ if (parsedCli.hasOption(HELP_CMD)) {
+ printUsage();
+ return 0;
+ }
+
+ boolean hasOption = false;
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ try {
+ if (parsedCli.hasOption(ADD_QUEUES_OPTION)) {
+ hasOption = true;
+ addQueues(parsedCli.getOptionValue(ADD_QUEUES_OPTION), updateInfo);
+ }
+ if (parsedCli.hasOption(REMOVE_QUEUES_OPTION)) {
+ hasOption = true;
+ removeQueues(parsedCli.getOptionValue(REMOVE_QUEUES_OPTION),
+ updateInfo);
+ }
+ if (parsedCli.hasOption(UPDATE_QUEUES_OPTION)) {
+ hasOption = true;
+ updateQueues(parsedCli.getOptionValue(UPDATE_QUEUES_OPTION),
+ updateInfo);
+ }
+ if (parsedCli.hasOption(GLOBAL_OPTIONS)) {
+ hasOption = true;
+ globalUpdates(parsedCli.getOptionValue(GLOBAL_OPTIONS), updateInfo);
+ }
+ } catch (IllegalArgumentException e) {
+ System.err.println(e.getMessage());
+ return -1;
+ }
+
+ if (!hasOption) {
+ System.err.println("Invalid Command Usage: ");
+ printUsage();
+ return -1;
+ }
+
+ Client webServiceClient = Client.create();
+ WebResource webResource = webServiceClient.resource(WebAppUtils.
+ getRMWebAppURLWithScheme(getConf()));
+ ClientResponse response = webResource.path("ws").path("v1").path("cluster")
+ .path("sched-conf").accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ if (response != null) {
+ if (response.getStatus() == Status.OK.getStatusCode()) {
+ System.out.println("Configuration changed successfully.");
+ return 0;
+ } else {
+ System.err.println("Configuration change unsuccessful: "
+ + response.getEntity(String.class));
+ }
+ } else {
+ System.err.println("Configuration change unsuccessful: null response");
+ }
+ return -1;
+ }
+
+ @VisibleForTesting
+ void addQueues(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ ArrayList<QueueConfigInfo> queueConfigInfos = new ArrayList<>();
+ for (String arg : args.split(";")) {
+ queueConfigInfos.add(getQueueConfigInfo(arg));
+ }
+ updateInfo.setAddQueueInfo(queueConfigInfos);
+ }
+
+ @VisibleForTesting
+ void removeQueues(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ List<String> queuesToRemove = Arrays.asList(args.split(","));
+ updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove));
+ }
+
+ @VisibleForTesting
+ void updateQueues(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ ArrayList<QueueConfigInfo> queueConfigInfos = new ArrayList<>();
+ for (String arg : args.split(";")) {
+ queueConfigInfos.add(getQueueConfigInfo(arg));
+ }
+ updateInfo.setUpdateQueueInfo(queueConfigInfos);
+ }
+
+ @VisibleForTesting
+ void globalUpdates(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ HashMap<String, String> globalUpdates = new HashMap<>();
+ for (String globalUpdate : args.split(",")) {
+ putKeyValuePair(globalUpdates, globalUpdate);
+ }
+ updateInfo.setGlobalParams(globalUpdates);
+ }
+
+ private QueueConfigInfo getQueueConfigInfo(String arg) {
+ String[] queueArgs = arg.split(",");
+ String queuePath = queueArgs[0];
+ Map<String, String> queueConfigs = new HashMap<>();
+ for (int i = 1; i < queueArgs.length; ++i) {
+ putKeyValuePair(queueConfigs, queueArgs[i]);
+ }
+ return new QueueConfigInfo(queuePath, queueConfigs);
+ }
+
+ private void putKeyValuePair(Map<String, String> kv, String args) {
+ String[] argParts = args.split("=");
+ if (argParts.length == 1) {
+ if (argParts[0].isEmpty() || !args.contains("=")) {
+ throw new IllegalArgumentException(CONF_ERR_MSG);
+ } else {
+ // key specified, but no value e.g. "confKey="
+ kv.put(argParts[0], null);
+ }
+ } else if (argParts.length > 2) {
+ throw new IllegalArgumentException(CONF_ERR_MSG);
+ } else {
+ if (argParts[0].isEmpty()) {
+ throw new IllegalArgumentException(CONF_ERR_MSG);
+ }
+ kv.put(argParts[0], argParts[1]);
+ }
+ }
+
+ private void printUsage() {
+ System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1,"
+ + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] "
+ + "[-remove queueRemovePath1,queueRemovePath2] "
+ + "[-update queueUpdatePath1,confKey1=confVal1] "
+ + "[-global globalConfKey1=globalConfVal1,"
+ + "globalConfKey2=globalConfVal2]");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
new file mode 100644
index 0000000..d2f0639
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -0,0 +1,160 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.client.cli;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Class for testing {@link SchedConfCLI}.
+ */
+public class TestSchedConfCLI {
+
+ private ByteArrayOutputStream sysOutStream;
+ private PrintStream sysOut;
+
+ private ByteArrayOutputStream sysErrStream;
+ private PrintStream sysErr;
+
+ private SchedConfCLI cli;
+
+ @Before
+ public void setUp() {
+ sysOutStream = new ByteArrayOutputStream();
+ sysOut = new PrintStream(sysOutStream);
+ System.setOut(sysOut);
+
+ sysErrStream = new ByteArrayOutputStream();
+ sysErr = new PrintStream(sysErrStream);
+ System.setErr(sysErr);
+
+ cli = new SchedConfCLI();
+ }
+
+ @Test(timeout = 10000)
+ public void testInvalidConf() throws Exception {
+ // conf pair with no key should be invalid
+ int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+ exitCode = cli.run(new String[] {"-update", "root.a,=confVal"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+
+ exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+ exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+ }
+
+ @Test(timeout = 10000)
+ public void testAddQueues() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.addQueues("root.a,a1=aVal1,a2=aVal2," +
+ "a3=", schedUpdateInfo);
+ QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0);
+ assertEquals("root.a", addInfo.getQueue());
+ Map<String, String> params = addInfo.getParams();
+ assertEquals(3, params.size());
+ assertEquals("aVal1", params.get("a1"));
+ assertEquals("aVal2", params.get("a2"));
+ assertNull(params.get("a3"));
+
+ schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ assertEquals(2, schedUpdateInfo.getAddQueueInfo().size());
+ QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0);
+ assertEquals("root.b", bAddInfo.getQueue());
+ Map<String, String> bParams = bAddInfo.getParams();
+ assertEquals(1, bParams.size());
+ assertEquals("bVal1", bParams.get("b1"));
+ QueueConfigInfo cAddInfo = schedUpdateInfo.getAddQueueInfo().get(1);
+ assertEquals("root.c", cAddInfo.getQueue());
+ Map<String, String> cParams = cAddInfo.getParams();
+ assertEquals(1, cParams.size());
+ assertEquals("cVal1", cParams.get("c1"));
+ }
+
+ @Test(timeout = 10000)
+ public void testRemoveQueues() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo);
+ List<String> removeInfo = schedUpdateInfo.getRemoveQueueInfo();
+ assertEquals(3, removeInfo.size());
+ assertEquals("root.a", removeInfo.get(0));
+ assertEquals("root.b", removeInfo.get(1));
+ assertEquals("root.c.c1", removeInfo.get(2));
+ }
+
+ @Test(timeout = 10000)
+ public void testUpdateQueues() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.updateQueues("root.a,a1=aVal1,a2=aVal2," +
+ "a3=", schedUpdateInfo);
+ QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
+ assertEquals("root.a", updateInfo.getQueue());
+ Map<String, String> params = updateInfo.getParams();
+ assertEquals(3, params.size());
+ assertEquals("aVal1", params.get("a1"));
+ assertEquals("aVal2", params.get("a2"));
+ assertNull(params.get("a3"));
+
+ schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size());
+ QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
+ assertEquals("root.b", bUpdateInfo.getQueue());
+ Map<String, String> bParams = bUpdateInfo.getParams();
+ assertEquals(1, bParams.size());
+ assertEquals("bVal1", bParams.get("b1"));
+ QueueConfigInfo cUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(1);
+ assertEquals("root.c", cUpdateInfo.getQueue());
+ Map<String, String> cParams = cUpdateInfo.getParams();
+ assertEquals(1, cParams.size());
+ assertEquals("cVal1", cParams.get("c1"));
+ }
+
+ @Test(timeout = 10000)
+ public void testGlobalUpdate() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.globalUpdates("schedKey1=schedVal1,schedKey2=schedVal2",
+ schedUpdateInfo);
+ Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams();
+ assertEquals(2, globalInfo.size());
+ assertEquals("schedVal1", globalInfo.get("schedKey1"));
+ assertEquals("schedVal2", globalInfo.get("schedKey2"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
new file mode 100644
index 0000000..aec6762
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Data structures for scheduler configuration mutation info.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.webapp.dao;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
index 4167e21..1cf1e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
@@ -23,9 +23,14 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
import org.apache.hadoop.conf.Configuration;
import org.codehaus.jettison.json.JSONObject;
+import java.io.StringWriter;
+
/**
* This class contains several utility function which could be used to generate
* Restful calls to RM/NM/AHS.
@@ -59,4 +64,13 @@ public final class YarnWebServiceUtils {
.get(ClientResponse.class);
return response.getEntity(JSONObject.class);
}
+
+ @SuppressWarnings("rawtypes")
+ public static String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
index 3a388fe..5bc5874 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
/**
* Interface for determining whether configuration mutations are allowed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
index 6648668..1de6f6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
/**
* Default configuration mutation ACL policy. Checks if user is YARN admin.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 027d944..007dc29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 6b8306c..86be7c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 4d367e0..da395b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,11 +137,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index eb97260..670c0f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
index 0a82d50..ee53fd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index b97e212..1da4e65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -199,6 +199,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
index b20eda6..d1d91c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+package org.apache.hadoop.yarn.webapp.dao;
import java.util.HashMap;
import java.util.Map;
@@ -54,4 +54,4 @@ public class QueueConfigInfo {
return this.params;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
index b7c585e..bb84096 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+package org.apache.hadoop.yarn.webapp.dao;
import java.util.ArrayList;
import java.util.HashMap;
@@ -54,16 +54,32 @@ public class SchedConfUpdateInfo {
return addQueueInfo;
}
+ public void setAddQueueInfo(ArrayList<QueueConfigInfo> addQueueInfo) {
+ this.addQueueInfo = addQueueInfo;
+ }
+
public ArrayList<String> getRemoveQueueInfo() {
return removeQueueInfo;
}
+ public void setRemoveQueueInfo(ArrayList<String> removeQueueInfo) {
+ this.removeQueueInfo = removeQueueInfo;
+ }
+
public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
return updateQueueInfo;
}
+ public void setUpdateQueueInfo(ArrayList<QueueConfigInfo> updateQueueInfo) {
+ this.updateQueueInfo = updateQueueInfo;
+ }
+
@XmlElementWrapper(name = "global-updates")
public HashMap<String, String> getGlobalParams() {
return global;
}
+
+ public void setGlobalParams(HashMap<String, String> globalInfo) {
+ this.global = globalInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
index 0f5a3d8..398e909 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 3216781..9104f16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/916bdbd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 5fbe36f..26ef1b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -22,8 +22,6 @@ import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.json.JSONJAXBContext;
-import com.sun.jersey.api.json.JSONMarshaller;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
@@ -35,11 +33,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -50,7 +49,6 @@ import javax.ws.rs.core.Response.Status;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.StringWriter;
import java.util.HashMap;
import java.util.Map;
@@ -183,8 +181,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -218,8 +216,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -244,8 +242,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -269,8 +267,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -300,8 +298,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -332,8 +330,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -360,8 +358,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf =
@@ -395,8 +393,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
@@ -413,8 +411,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
newCSConf = cs.getConfiguration();
@@ -443,8 +441,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf =
@@ -468,8 +466,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf =
@@ -483,8 +481,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
newCSConf =
@@ -506,13 +504,4 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
super.tearDown();
}
-
- @SuppressWarnings("rawtypes")
- private String toJson(Object nsli, Class klass) throws Exception {
- StringWriter sw = new StringWriter();
- JSONJAXBContext ctx = new JSONJAXBContext(klass);
- JSONMarshaller jm = ctx.createJSONMarshaller();
- jm.marshallToJSON(nsli, sw);
- return sw.toString();
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-12627. Fix typo in DFSAdmin
command output. Contributed by Ajay Kumar.
Posted by su...@apache.org.
HDFS-12627. Fix typo in DFSAdmin command output. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb0a742a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb0a742a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb0a742a
Branch: refs/heads/YARN-5881
Commit: bb0a742aac1dc03d08beff3cb4b7b04b8036fdcc
Parents: fa5cfc6
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Oct 11 12:26:14 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Oct 11 12:26:14 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 4 ++--
.../hadoop/hdfs/TestSnapshotCommands.java | 22 ++++++++++++--------
.../src/test/resources/testHDFSConf.xml | 4 ++--
3 files changed, 17 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb0a742a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 9805972..6b4edd5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -753,7 +753,7 @@ public class DFSAdmin extends FsShell {
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
- System.out.println("Allowing snaphot on " + argv[1] + " succeeded");
+ System.out.println("Allowing snapshot on " + argv[1] + " succeeded");
}
/**
@@ -770,7 +770,7 @@ public class DFSAdmin extends FsShell {
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
- System.out.println("Disallowing snaphot on " + argv[1] + " succeeded");
+ System.out.println("Disallowing snapshot on " + argv[1] + " succeeded");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb0a742a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
index 7e1af2a..8d667f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
@@ -85,10 +85,12 @@ public class TestSnapshotCommands {
@Test
public void testAllowSnapshot() throws Exception {
// Idempotent test
- DFSTestUtil.DFSAdminRun("-allowSnapshot /sub1", 0, "Allowing snaphot on /sub1 succeeded", conf);
- // allow normal dir success
+ DFSTestUtil.DFSAdminRun("-allowSnapshot /sub1", 0,
+ "Allowing snapshot " + "on /sub1 succeeded", conf);
+ // allow normal dir success
DFSTestUtil.FsShellRun("-mkdir /sub2", conf);
- DFSTestUtil.DFSAdminRun("-allowSnapshot /sub2", 0, "Allowing snaphot on /sub2 succeeded", conf);
+ DFSTestUtil.DFSAdminRun("-allowSnapshot /sub2", 0,
+ "Allowing snapshot " + "on /sub2 succeeded", conf);
// allow non-exists dir failed
DFSTestUtil.DFSAdminRun("-allowSnapshot /sub3", -1, null, conf);
}
@@ -159,9 +161,11 @@ public class TestSnapshotCommands {
DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", -1,
"disallowSnapshot: The directory /sub1 has snapshot(s). Please redo the operation after removing all the snapshots.", conf);
DFSTestUtil.FsShellRun("-deleteSnapshot /sub1 sn1", conf);
- DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", 0, "Disallowing snaphot on /sub1 succeeded", conf);
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", 0,
+ "Disallowing snapshot on /sub1 succeeded", conf);
// Idempotent test
- DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", 0, "Disallowing snaphot on /sub1 succeeded", conf);
+ DFSTestUtil.DFSAdminRun("-disallowSnapshot /sub1", 0,
+ "Disallowing snapshot on /sub1 succeeded", conf);
// now it can be deleted
DFSTestUtil.FsShellRun("-rmr /sub1", conf);
}
@@ -173,7 +177,7 @@ public class TestSnapshotCommands {
config.set("fs.defaultFS", "hdfs://127.0.0.1:1024");
String path = fs.getUri() + "/Fully/QPath";
DFSTestUtil.DFSAdminRun("-allowSnapshot " + path, 0,
- "Allowing snaphot on " + path + " succeeded", config);
+ "Allowing snapshot on " + path + " succeeded", config);
DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn1", config);
// create file1
DFSTestUtil
@@ -193,7 +197,7 @@ public class TestSnapshotCommands {
DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn1", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn3", config);
DFSTestUtil.DFSAdminRun("-disallowSnapshot " + path, 0,
- "Disallowing snaphot on " + path + " succeeded", config);
+ "Disallowing snapshot on " + path + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true);
}
@@ -205,7 +209,7 @@ public class TestSnapshotCommands {
fs.mkdirs(snapDirPath);
DFSTestUtil.DFSAdminRun("-allowSnapshot " + snapDirPath, 0,
- "Allowing snaphot on " + snapDirPath + " succeeded", config);
+ "Allowing snapshot on " + snapDirPath + " succeeded", config);
DFSTestUtil.createFile(fs, new Path(snapDirPath, "file1"),
1024, (short) 1, 100);
DFSTestUtil.FsShellRun("-createSnapshot " + snapDirPath + " sn1", config);
@@ -242,7 +246,7 @@ public class TestSnapshotCommands {
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn3", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + snapDir + " sn4", config);
DFSTestUtil.DFSAdminRun("-disallowSnapshot " + snapDir, 0,
- "Disallowing snaphot on " + snapDirPath + " succeeded", config);
+ "Disallowing snapshot on " + snapDirPath + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb0a742a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index ba90efa..5094183 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -16863,7 +16863,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>Allowing snaphot on /user/USERNAME/dir1 succeeded</expected-output>
+ <expected-output>Allowing snapshot on /user/USERNAME/dir1 succeeded</expected-output>
</comparator>
</comparators>
</test>
@@ -16881,7 +16881,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>Disallowing snaphot on /user/USERNAME/dir1 succeeded</expected-output>
+ <expected-output>Disallowing snapshot on /user/USERNAME/dir1 succeeded</expected-output>
</comparator>
</comparators>
</test>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-12542. Update javadoc and
documentation for listStatus. Contributed by Ajay Kumar.
Posted by su...@apache.org.
HDFS-12542. Update javadoc and documentation for listStatus. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8acdf5c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8acdf5c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8acdf5c2
Branch: refs/heads/YARN-5881
Commit: 8acdf5c2742c081f3e0e96e13eb940a39964a58f
Parents: bb0a742
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Oct 11 12:29:35 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Oct 11 12:29:35 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 16 ++++++++++++++++
.../hadoop/fs/http/client/HttpFSFileSystem.java | 18 ++++++++++++------
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 5 +++++
3 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8acdf5c2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index d3a8b23..0fb6f22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1495,6 +1495,15 @@ public class WebHdfsFileSystem extends FileSystem
}
}
+ /**
+ * Get {@link FileStatus} of files/directories in the given path. If path
+ * corresponds to a file then {@link FileStatus} of that file is returned.
+ * Else if path represents a directory then {@link FileStatus} of all
+ * files/directories inside given path is returned.
+ *
+ * @param f given path
+ * @return the statuses of the files/directories in the given path
+ */
@Override
public FileStatus[] listStatus(final Path f) throws IOException {
statistics.incrementReadOps(1);
@@ -1519,6 +1528,13 @@ public class WebHdfsFileSystem extends FileSystem
private static final byte[] EMPTY_ARRAY = new byte[] {};
+ /**
+ * Get DirectoryEntries of the given path. DirectoryEntries contains an array
+ * of {@link FileStatus}, as well as iteration information.
+ *
+ * @param f given path
+ * @return DirectoryEntries for given path
+ */
@Override
public DirectoryEntries listStatusBatch(Path f, byte[] token) throws
FileNotFoundException, IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8acdf5c2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 3ff7a61..ffd64a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -705,14 +705,13 @@ public class HttpFSFileSystem extends FileSystem
}
/**
- * List the statuses of the files/directories in the given path if the path is
- * a directory.
+ * Get {@link FileStatus} of files/directories in the given path. If path
+ * corresponds to a file then {@link FileStatus} of that file is returned.
+ * Else if path represents a directory then {@link FileStatus} of all
+ * files/directories inside given path is returned.
*
* @param f given path
- *
- * @return the statuses of the files/directories in the given patch
- *
- * @throws IOException
+ * @return the statuses of the files/directories in the given path
*/
@Override
public FileStatus[] listStatus(Path f) throws IOException {
@@ -725,6 +724,13 @@ public class HttpFSFileSystem extends FileSystem
return toFileStatuses(json, f);
}
+ /**
+ * Get {@link DirectoryEntries} of the given path. {@link DirectoryEntries}
+ * contains an array of {@link FileStatus}, as well as iteration information.
+ *
+ * @param f given path
+ * @return {@link DirectoryEntries} for given path
+ */
@Override
public DirectoryEntries listStatusBatch(Path f, byte[] token) throws
FileNotFoundException, IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8acdf5c2/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 73c6558..4a1395e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -467,6 +467,8 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
{
"accessTime" : 1320171722771,
"blockSize" : 33554432,
+ "childrenNum" : 0,
+ "fileId" : 16388,
"group" : "supergroup",
"length" : 24930,
"modificationTime": 1320171722771,
@@ -474,11 +476,14 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
"pathSuffix" : "a.patch",
"permission" : "644",
"replication" : 1,
+ "storagePolicy" : 0,
"type" : "FILE"
},
{
"accessTime" : 0,
"blockSize" : 0,
+ "childrenNum" : 0,
+ "fileId" : 16389,
"group" : "supergroup",
"length" : 0,
"modificationTime": 1320895981256,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-14913. Sticky bit
implementation for rename() operation in Azure WASB. Contributed by Varada
Hemeswari.
Posted by su...@apache.org.
HADOOP-14913. Sticky bit implementation for rename() operation in Azure WASB.
Contributed by Varada Hemeswari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3de57441
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3de57441
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3de57441
Branch: refs/heads/YARN-5881
Commit: 3de574413c4d5554213d02bd0ad343ba82cf82aa
Parents: ebb34c7
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 11 18:06:43 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 11 18:06:43 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/contract/ContractTestUtils.java | 21 ++
.../hadoop/fs/azure/NativeAzureFileSystem.java | 67 +++-
.../TestNativeAzureFileSystemAuthorization.java | 338 ++++++++++++++++++-
3 files changed, 406 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3de57441/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index 8c01d2b..4dd9c3f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -671,6 +671,27 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Execute a {@link FileSystem#rename(Path, Path)}, and verify that the
+ * outcome was as expected. There is no preflight checking of arguments;
+ * everything is left to the rename() command.
+ * @param fs filesystem
+ * @param source source path
+ * @param dest destination path
+ * @param expectedResult expected return code
+ * @throws IOException on any IO failure.
+ */
+ public static void assertRenameOutcome(FileSystem fs,
+ Path source,
+ Path dest,
+ boolean expectedResult) throws IOException {
+ boolean result = fs.rename(source, dest);
+ if (expectedResult != result) {
+ fail(String.format("Expected rename(%s, %s) to return %b,"
+ + " but result was %b", source, dest, expectedResult, result));
+ }
+ }
+
+ /**
* Read in "length" bytes, convert to an ascii string.
* @param fs filesystem
* @param path path to read
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3de57441/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 07cfe72..9effb3b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -3154,8 +3154,6 @@ public class NativeAzureFileSystem extends FileSystem {
return false;
}
- performAuthCheck(srcParentFolder, WasbAuthorizationOperations.WRITE, "rename", absoluteSrcPath);
-
String srcKey = pathToKey(absoluteSrcPath);
if (srcKey.length() == 0) {
@@ -3163,12 +3161,30 @@ public class NativeAzureFileSystem extends FileSystem {
return false;
}
+ performAuthCheck(srcParentFolder, WasbAuthorizationOperations.WRITE, "rename",
+ absoluteSrcPath);
+
+ if (this.azureAuthorization) {
+ try {
+ performStickyBitCheckForRenameOperation(absoluteSrcPath, srcParentFolder);
+ } catch (FileNotFoundException ex) {
+ return false;
+ } catch (IOException ex) {
+ Throwable innerException = checkForAzureStorageException(ex);
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ LOG.debug("Encountered FileNotFound Exception when performing sticky bit check "
+ + "on {}. Failing rename", srcKey);
+ return false;
+ }
+ throw ex;
+ }
+ }
+
// Figure out the final destination
Path absoluteDstPath = makeAbsolute(dst);
Path dstParentFolder = absoluteDstPath.getParent();
- performAuthCheck(dstParentFolder, WasbAuthorizationOperations.WRITE, "rename", absoluteDstPath);
-
String dstKey = pathToKey(absoluteDstPath);
FileMetadata dstMetadata = null;
try {
@@ -3193,6 +3209,9 @@ public class NativeAzureFileSystem extends FileSystem {
if (dstMetadata != null && dstMetadata.isDir()) {
// It's an existing directory.
+ performAuthCheck(absoluteDstPath, WasbAuthorizationOperations.WRITE, "rename",
+ absoluteDstPath);
+
dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
LOG.debug("Destination {} "
+ " is a directory, adjusted the destination to be {}", dst, dstKey);
@@ -3228,6 +3247,9 @@ public class NativeAzureFileSystem extends FileSystem {
LOG.debug("Parent of the destination {}"
+ " is a file, failing the rename.", dst);
return false;
+ } else {
+ performAuthCheck(dstParentFolder, WasbAuthorizationOperations.WRITE,
+ "rename", absoluteDstPath);
}
}
FileMetadata srcMetadata = null;
@@ -3406,6 +3428,43 @@ public class NativeAzureFileSystem extends FileSystem {
}
/**
+ * Performs sticky bit check on source folder for rename operation.
+ *
+ * @param srcPath - path which is to be renamed.
+ * @param srcParentPath - parent to srcPath to check for stickybit check.
+ * @throws FileNotFoundException if srcPath or srcParentPath do not exist.
+ * @throws WasbAuthorizationException if stickybit check is violated.
+ * @throws IOException when retrieving metadata operation fails.
+ */
+ private void performStickyBitCheckForRenameOperation(Path srcPath,
+ Path srcParentPath)
+ throws FileNotFoundException, WasbAuthorizationException, IOException {
+
+ String srcKey = pathToKey(srcPath);
+ FileMetadata srcMetadata = null;
+ srcMetadata = store.retrieveMetadata(srcKey);
+ if (srcMetadata == null) {
+ LOG.debug("Source {} doesn't exist. Failing rename.", srcPath);
+ throw new FileNotFoundException(
+ String.format("%s does not exist.", srcPath));
+ }
+
+ String parentkey = pathToKey(srcParentPath);
+ FileMetadata parentMetadata = store.retrieveMetadata(parentkey);
+ if (parentMetadata == null) {
+ LOG.debug("Path {} doesn't exist, failing rename.", srcParentPath);
+ throw new FileNotFoundException(
+ String.format("%s does not exist.", parentkey));
+ }
+
+ if (isStickyBitCheckViolated(srcMetadata, parentMetadata)) {
+ throw new WasbAuthorizationException(
+ String.format("Rename operation for %s is not permitted."
+ + " Details : Stickybit check failed.", srcPath));
+ }
+ }
+
+ /**
* Return an array containing hostnames, offset and size of
* portions of the given file. For WASB we'll just lie and give
* fake hosts to make sure we get many splits in MR jobs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3de57441/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index 9a5e6e0..c8d6507 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -45,6 +45,7 @@ import com.google.common.annotations.VisibleForTesting;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.junit.Assert.assertEquals;
/**
@@ -69,6 +70,7 @@ public class TestNativeAzureFileSystemAuthorization
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true");
+ conf.set(KEY_USE_SECURE_MODE, "true");
conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URLS, "http://localhost/");
conf.set(NativeAzureFileSystem.AZURE_CHOWN_USERLIST_PROPERTY_NAME, "user1 , user2");
conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "false");
@@ -333,15 +335,14 @@ public class TestNativeAzureFileSystemAuthorization
fs.updateWasbAuthorizer(authorizer);
try {
- fs.create(srcPath);
- ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath);
- fs.rename(srcPath, dstPath);
- ContractTestUtils.assertPathExists(fs, "destPath does not exist", dstPath);
- ContractTestUtils.assertPathDoesNotExist(fs, "sourcePath exists after rename!", srcPath);
+ touch(fs, srcPath);
+ assertPathExists(fs, "sourcePath does not exist", srcPath);
+ assertRenameOutcome(fs, srcPath, dstPath, true);
+ assertPathExists(fs, "destPath does not exist", dstPath);
+ assertPathDoesNotExist(fs, "sourcePath exists after rename!", srcPath);
}
finally {
- allowRecursiveDelete(fs, parentDir.toString());
- fs.delete(parentDir, true);
+ recursiveDelete(parentDir);
}
}
@@ -399,14 +400,14 @@ public class TestNativeAzureFileSystemAuthorization
fs.updateWasbAuthorizer(authorizer);
try {
- fs.create(srcPath);
+ touch(fs, srcPath);
ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath);
+ fs.mkdirs(parentDstDir);
fs.rename(srcPath, dstPath);
ContractTestUtils.assertPathDoesNotExist(fs, "destPath does not exist", dstPath);
} finally {
ContractTestUtils.assertPathExists(fs, "sourcePath does not exist after rename !", srcPath);
- allowRecursiveDelete(fs, parentSrcDir.toString());
- fs.delete(parentSrcDir, true);
+ recursiveDelete(parentSrcDir);
}
}
@@ -430,18 +431,323 @@ public class TestNativeAzureFileSystemAuthorization
fs.updateWasbAuthorizer(authorizer);
try {
- fs.create(srcPath);
+ touch(fs, srcPath);
ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", srcPath);
fs.mkdirs(parentDstDir);
- fs.rename(srcPath, dstPath);
+ assertRenameOutcome(fs, srcPath, dstPath, true);
ContractTestUtils.assertPathDoesNotExist(fs, "sourcePath does not exist", srcPath);
ContractTestUtils.assertPathExists(fs, "destPath does not exist", dstPath);
} finally {
- allowRecursiveDelete(fs, parentSrcDir.toString());
- fs.delete(parentSrcDir, true);
+ recursiveDelete(parentSrcDir);
+ recursiveDelete(parentDstDir);
+ }
+ }
+
+ /**
+ * Recursive delete for teardown/finally operations, setting the permissions
+ * to do the delete before invoking FileSystem.delete.
+ * Exceptions are caught and logged at ERROR.
+ * @param path path to delete
+ */
+ private void recursiveDelete(Path path) {
+ try {
+ allowRecursiveDelete(fs, path.toString());
+ fs.delete(path, true);
+ } catch (IOException e) {
+ LOG.error("Failed to delete {}", path, e);
+ }
+ }
+
+ /**
+ * Positive test to check rename succeeds for hierarchy of
+ * files and folders under a src directory when destination
+ * folder already exists.
+ */
+ @Test
+ public void testRenamePositiveWhenDestinationFolderExists() throws Throwable {
+
+ Path parentSrcDir = new Path("/testRenamePositiveForFolderSrc");
+ Path srcFilePath = new Path(parentSrcDir, "test1.dat");
+ Path srcFolderPath = new Path(parentSrcDir, "testFolder");
+ Path dstDir = new Path("/testRenamePositiveForFolderDst");
+ Path finalDstDir = new Path(dstDir, "testRenamePositiveForFolderSrc");
+ Path dstFilePath = new Path(finalDstDir, "test1.dat");
+ Path dstFolderPath = new Path(finalDstDir, "testFolder");
+
+ /* to create parent dirs */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(dstDir.toString(), WRITE, true);
+ /* Required for assertPathExists calls */
+ authorizer.addAuthRuleForOwner("/", READ, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(finalDstDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ touch(fs, srcFilePath);
+ assertPathExists(fs, "srcFilePath does not exist", srcFilePath);
+ fs.mkdirs(srcFolderPath);
+ assertIsDirectory(fs, srcFolderPath);
+ fs.mkdirs(dstDir);
+ assertIsDirectory(fs, dstDir);
+ assertRenameOutcome(fs, parentSrcDir, dstDir, true);
+ assertPathDoesNotExist(fs, "parentSrcDir exists", parentSrcDir);
+ assertPathDoesNotExist(fs, "srcFilePath exists", srcFilePath);
+ assertPathDoesNotExist(fs, "srcFolderPath exists", srcFolderPath);
+ assertPathExists(fs, "destPath does not exist", dstDir);
+ assertPathExists(fs, "dstFilePath does not exist", dstFilePath);
+ assertPathExists(fs, "dstFolderPath does not exist", dstFolderPath);
+ } finally {
+ recursiveDelete(parentSrcDir);
+ recursiveDelete(dstDir);
+ }
+ }
+
+ /**
+ * Positive test to check rename succeeds for hierarchy of
+ * files and folders under a src directory and when the destination
+ * folder does not exist.
+ */
+ @Test
+ public void testRenamePositiveWhenDestinationFolderDoesNotExist() throws Throwable {
+ Path srcParentDir = new Path("/testRenamePositiveWhenDestinationFolderDoesNotExist");
+ Path srcDir = new Path(srcParentDir, "srcDir");
+ Path srcFilePath = new Path(srcDir, "test1.dat");
+ Path srcSubDirPath = new Path(srcDir, "testFolder");
+ Path srcSubDirFilePath = new Path(srcSubDirPath, "test2.dat");
+ Path dstDir = new Path(srcParentDir, "dstDir");
+ Path dstFilePath = new Path(dstDir, "test1.dat");
+ Path dstSubDirPath = new Path(dstDir, "testFolder");
+ Path dstSubDirFilePath = new Path(dstSubDirPath, "test2.dat");
+
+ /* to create parent dirs */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(srcParentDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(srcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(srcSubDirPath.toString(), WRITE, true);
+ /* Required for asserPathExists calls */
+ authorizer.addAuthRuleForOwner("/", READ, true);
+ authorizer.addAuthRuleForOwner(srcParentDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(srcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(srcSubDirPath.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(dstDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(dstSubDirPath.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ touch(fs, srcFilePath);
+ assertPathExists(fs, "srcFilePath does not exist", srcFilePath);
+ fs.mkdirs(srcSubDirPath);
+ assertIsDirectory(fs, srcSubDirPath);
+ touch(fs, srcSubDirFilePath);
+ assertPathExists(fs, "srcSubDirFilePath does not exist", srcSubDirFilePath);
+ assertRenameOutcome(fs, srcDir, dstDir, true);
+ assertPathDoesNotExist(fs, "srcDir exists", srcDir);
+ assertPathDoesNotExist(fs, "srcFilePath exists", srcFilePath);
+ assertPathDoesNotExist(fs, "srcSubDirPath exists", srcSubDirPath);
+ assertPathDoesNotExist(fs, "srcSubDirFilePath exists", srcSubDirFilePath);
+ assertPathExists(fs, "destPath does not exist", dstDir);
+ assertPathExists(fs, "dstFilePath does not exist", dstFilePath);
+ assertPathExists(fs, "dstSubDirPath does not exist", dstSubDirPath);
+ assertPathExists(fs, "dstSubDirFilePath does not exist", dstSubDirFilePath);
+ } finally {
+ recursiveDelete(srcParentDir);
+ }
+ }
+
+ /**
+ * Test to verify rename fails and returns false when
+ * the source to be renamed does not exist.
+ */
+ @Test
+ public void testRenameOnNonExistentSource() throws Throwable {
+
+ Path parentSrcDir = new Path("/testRenameOnNonExistentSourceFolderSrc");
+ Path srcPath = new Path(parentSrcDir, "test1.dat");
+ Path parentDstDir = new Path("/testRenameOnNonExistentSourceFolderDst");
+ Path dstPath = new Path(parentDstDir, "test2.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dirs */
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
+ // required for assertpathExists calls
+ authorizer.addAuthRuleForOwner("/", READ, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.mkdirs(parentSrcDir);
+ assertIsDirectory(fs, parentSrcDir);
+ fs.mkdirs(parentDstDir);
+ // should return false
+ assertRenameOutcome(fs, srcPath, dstPath, false);
+ assertPathDoesNotExist(fs, "destPath exists!", dstPath);
+ } finally {
+ recursiveDelete(parentSrcDir);
+ recursiveDelete(parentDstDir);
+ }
+ }
+
+ /**
+ * Positive test to check rename succeeds when sticky bit is set on
+ * source parent directory and user owns the source directory.
+ */
+ @Test
+ public void testRenameWithStickyBitPositive() throws Throwable {
+
+ Path parentSrcDir = new Path("/testRenameWithStickyBitPositiveSrc");
+ Path srcPath = new Path(parentSrcDir, "test1.dat");
+ Path parentDstDir = new Path("/testRenameWithStickyBitPositiveDst");
+ Path dstPath = new Path(parentDstDir, "test2.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dirs */
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
+ /* Required for asserPathExists calls */
+ authorizer.addAuthRuleForOwner("/", READ, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ touch(fs, srcPath);
+ assertPathExists(fs, "sourcePath does not exist", srcPath);
+ fs.mkdirs(parentDstDir);
+ assertIsDirectory(fs, parentDstDir);
+ // set stickybit on parent directory
+ fs.setPermission(parentSrcDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+ assertRenameOutcome(fs, srcPath, dstPath, true);
+ assertPathDoesNotExist(fs, "sourcePath exists", srcPath);
+ assertPathExists(fs, "destPath does not exist", dstPath);
+ } finally {
+ recursiveDelete(parentSrcDir);
+ recursiveDelete(parentDstDir);
+ }
+ }
+
+ /**
+ * Test to check rename fails when sticky bit is set on
+ * parent of source directory and the user is not owner
+ * of parent or the source directory.
+ */
+ @Test
+ public void testRenameWithStickyBitNegative() throws Throwable {
+
+ final Path parentSrcDir = new Path("/testRenameWithStickyBitNegativeSrc");
+ final Path srcPath = new Path(parentSrcDir, "test1.dat");
+ final Path parentDstDir = new Path("/testRenameWithStickyBitNegativeDst");
+ final Path dstPath = new Path(parentDstDir, "test2.dat");
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage(String.format("Rename operation for %s is not permitted."
+ + " Details : Stickybit check failed.", srcPath.toString()));
+
+ /* to create parent dirs */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(),
+ WRITE, true);
+ /* Required for asserPathExists calls */
+ authorizer.addAuthRuleForOwner("/", READ, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ touch(fs, srcPath);
+ assertPathExists(fs, "sourcePath does not exist", srcPath);
+ fs.mkdirs(parentDstDir);
+ assertIsDirectory(fs, parentDstDir);
+ // set stickybit on parent of source folder
+ fs.setPermission(parentSrcDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Add auth rules for dummyuser
+ authorizer.addAuthRule(parentSrcDir.toString(),
+ WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDstDir.toString(),
+ WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentSrcDir.toString(),
+ READ, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDstDir.toString(),
+ READ, getCurrentUserShortName(), true);
+
+ try {
+ fs.rename(srcPath, dstPath);
+ } catch (WasbAuthorizationException wae) {
+ assertPathExists(fs, "sourcePath does not exist", srcPath);
+ assertPathDoesNotExist(fs, "destPath exists", dstPath);
+ throw wae;
+ }
+
+ return null;
+ }
+ });
+ } finally {
+ recursiveDelete(parentSrcDir);
+ recursiveDelete(parentDstDir);
+ }
+ }
+
+ /**
+ * Test to check rename returns false when sticky bit is set on
+ * parent of source parent directory and the source does not exist
+ */
+ @Test
+ public void testRenameOnNonExistentSourceWithStickyBit() throws Throwable {
- allowRecursiveDelete(fs, parentDstDir.toString());
- fs.delete(parentDstDir, true);
+ final Path parentSrcDir = new Path("/testRenameOnNonExistentSourceWithStickyBitSrc");
+ final Path srcPath = new Path(parentSrcDir, "test1.dat");
+ final Path parentDstDir = new Path("/testRenameOnNonExistentSourceWithStickyBitDest");
+ final Path dstPath = new Path(parentDstDir, "test2.dat");
+
+ /* to create parent dirs */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(),
+ WRITE, true);
+ /* Required for asserPathExists calls */
+ authorizer.addAuthRuleForOwner("/", READ, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.mkdirs(parentSrcDir);
+ assertIsDirectory(fs, parentSrcDir);
+ fs.mkdirs(parentDstDir);
+ assertIsDirectory(fs, parentDstDir);
+ // set stickybit on parent of source folder
+ fs.setPermission(parentSrcDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Add auth rules for dummyuser
+ authorizer.addAuthRule(parentSrcDir.toString(),
+ WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDstDir.toString(),
+ WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentSrcDir.toString(),
+ READ, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDstDir.toString(),
+ READ, getCurrentUserShortName(), true);
+ // should return false since srcPath does not exist.
+ assertRenameOutcome(fs, srcPath, dstPath, false);
+ assertPathDoesNotExist(fs, "destPath exists", dstPath);
+ return null;
+ }
+ });
+ } finally {
+ recursiveDelete(parentSrcDir);
+ recursiveDelete(parentDstDir);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-6322: Disable queue refresh
when configuration mutation is enabled. Contributed by Jonathan Hung
Posted by su...@apache.org.
YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c8b208a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c8b208a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c8b208a
Branch: refs/heads/YARN-5881
Commit: 4c8b208adb8c7639628676387e275856cf0842c9
Parents: 74ba6ff
Author: Xuan <xg...@apache.org>
Authored: Tue Aug 1 08:48:04 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../server/resourcemanager/AdminService.java | 12 +++++++++-
.../scheduler/MutableConfScheduler.java | 12 ++++++++--
.../scheduler/MutableConfigurationProvider.java | 4 +++-
.../scheduler/capacity/CapacityScheduler.java | 9 ++++++--
.../conf/MutableCSConfigurationProvider.java | 11 +++++-----
.../resourcemanager/TestRMAdminService.java | 23 ++++++++++++++++++++
.../TestMutableCSConfigurationProvider.java | 14 +++++++-----
7 files changed, 67 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 3457ae3..fd9e849 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.HAServiceProtocol;
@@ -92,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst
import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import com.google.common.annotations.VisibleForTesting;
@@ -384,6 +387,12 @@ public class AdminService extends CompositeService implements
RefreshQueuesResponse response =
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ if (scheduler instanceof MutableConfScheduler
+ && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+ throw new IOException("Scheduler configuration is mutable. " +
+ operation + " is not allowed in this scenario.");
+ }
refreshQueues();
RMAuditLogger.logSuccess(user.getShortUserName(), operation,
"AdminService");
@@ -393,7 +402,8 @@ public class AdminService extends CompositeService implements
}
}
- private void refreshQueues() throws IOException, YarnException {
+ @Private
+ public void refreshQueues() throws IOException, YarnException {
rm.getRMContext().getScheduler().reinitialize(getConfig(),
this.rm.getRMContext());
// refresh the reservation system
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 007dc29..313bf6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -33,10 +34,11 @@ public interface MutableConfScheduler extends ResourceScheduler {
* Update the scheduler's configuration.
* @param user Caller of this update
* @param confUpdate configuration update
- * @throws IOException if update is invalid
+ * @throws IOException if scheduler could not be reinitialized
+ * @throws YarnException if reservation system could not be reinitialized
*/
void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException;
+ SchedConfUpdateInfo confUpdate) throws IOException, YarnException;
/**
* Get the scheduler configuration.
@@ -50,4 +52,10 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @return the queue object
*/
Queue getQueue(String queueName);
+
+ /**
+ * Return whether the scheduler configuration is mutable.
+ * @return whether scheduler configuration is mutable or not.
+ */
+ boolean isConfigurationMutable();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 1f13467..9baf1ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -39,8 +40,9 @@ public interface MutableConfigurationProvider {
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
+ * @throws YarnException if reservation system could not be reinitialized
*/
void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
- confUpdate) throws IOException;
+ confUpdate) throws IOException, YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 6d2de7e..51ee6a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2620,8 +2620,8 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException {
- if (csConfProvider instanceof MutableConfigurationProvider) {
+ SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
+ if (isConfigurationMutable()) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
user, confUpdate);
} else {
@@ -2629,4 +2629,9 @@ public class CapacityScheduler extends
"provider does not support updating configuration.");
}
}
+
+ @Override
+ public boolean isConfigurationMutable() {
+ return csConfProvider instanceof MutableConfigurationProvider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 9ccc146..d03b2e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
@@ -58,7 +59,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private YarnConfigurationStore confStore;
private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
- private Configuration conf;
public MutableCSConfigurationProvider(RMContext rmContext) {
this.rmContext = rmContext;
@@ -96,7 +96,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
.getPolicy(config);
aclMutationPolicy.init(config, rmContext);
- this.conf = config;
}
@Override
@@ -109,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
@Override
public synchronized void mutateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
" queues.");
@@ -126,8 +125,8 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
}
try {
- rmContext.getScheduler().reinitialize(conf, rmContext);
- } catch (IOException e) {
+ rmContext.getRMAdminService().refreshQueues();
+ } catch (IOException | YarnException e) {
schedConf = oldConf;
confStore.confirmMutation(id, false);
throw e;
@@ -148,7 +147,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
}
try {
- rmContext.getScheduler().reinitialize(conf, rmContext);
+ rmContext.getScheduler().reinitialize(schedConf, rmContext);
} catch (IOException e) {
schedConf = oldConf;
confStore.confirmMutation(mutation.getId(), false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 588f16d..620ca58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -198,6 +198,29 @@ public class TestRMAdminService {
}
@Test
+ public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() {
+ configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ try {
+ rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
+ fail("Expected exception while calling refreshQueues when scheduler" +
+ " configuration is mutable.");
+ } catch (Exception ex) {
+ assertTrue(ex.getMessage().endsWith("Scheduler configuration is " +
+ "mutable. refreshQueues is not allowed in this scenario."));
+ }
+ }
+
+ @Test
public void testAdminRefreshNodesWithoutConfiguration()
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 9104f16..635a184 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -34,7 +36,6 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -50,6 +51,7 @@ public class TestMutableCSConfigurationProvider {
private SchedConfUpdateInfo goodUpdate;
private SchedConfUpdateInfo badUpdate;
private CapacityScheduler cs;
+ private AdminService adminService;
private static final UserGroupInformation TEST_USER = UserGroupInformation
.createUserForTesting("testUser", new String[] {});
@@ -61,6 +63,8 @@ public class TestMutableCSConfigurationProvider {
when(rmContext.getScheduler()).thenReturn(cs);
when(cs.getConfiguration()).thenReturn(
new CapacitySchedulerConfiguration());
+ adminService = mock(AdminService.class);
+ when(rmContext.getRMAdminService()).thenReturn(adminService);
confProvider = new MutableCSConfigurationProvider(rmContext);
goodUpdate = new SchedConfUpdateInfo();
Map<String, String> goodUpdateMap = new HashMap<>();
@@ -78,22 +82,20 @@ public class TestMutableCSConfigurationProvider {
}
@Test
- public void testInMemoryBackedProvider() throws IOException {
+ public void testInMemoryBackedProvider() throws IOException, YarnException {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
- doNothing().when(cs).reinitialize(any(Configuration.class),
- any(RMContext.class));
+ doNothing().when(adminService).refreshQueues();
confProvider.mutateConfiguration(TEST_USER, goodUpdate);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
- doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
- any(RMContext.class));
+ doThrow(new IOException()).when(adminService).refreshQueues();
try {
confProvider.mutateConfiguration(TEST_USER, badUpdate);
} catch (IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-12622. Fix enumerate in
HDFSErasureCoding.md. Contributed by Yiqun Lin.
Posted by su...@apache.org.
HDFS-12622. Fix enumerate in HDFSErasureCoding.md. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc63a6a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc63a6a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc63a6a5
Branch: refs/heads/YARN-5881
Commit: dc63a6a52b7fdf076ab83a774b0378f77c1c0cd3
Parents: 639f98c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 11 18:09:53 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 11 18:09:53 2017 +0900
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc63a6a5/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index c8ef6c7..a171665 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -140,6 +140,7 @@ Deployment
### Enable Intel ISA-L
HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps.
+
1. Build ISA-L library. Please refer to the official site "https://github.com/01org/isa-l/" for detail information.
2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop" in (BUILDING.txt) in the source code.
3. Use `-Dbundle.isal` to copy the contents of the `isal.lib` directory into the final tar file. Deploy Hadoop with the tar file. Make sure ISA-L is available on HDFS clients and DataNodes.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-6840. Implement zookeeper
based store for scheduler configuration updates. (Jonathan Hung via wangda)
Posted by su...@apache.org.
YARN-6840. Implement zookeeper based store for scheduler configuration updates. (Jonathan Hung via wangda)
Change-Id: I9debea674fe8c7e4109d4ca136965a1ea4c48bcc
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff39c0de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff39c0de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff39c0de
Branch: refs/heads/YARN-5881
Commit: ff39c0de206a4fce1f0e8a416357a7a8261f8634
Parents: 4d8abd8
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Sep 18 09:53:42 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 14 +-
.../src/main/resources/yarn-default.xml | 15 +-
.../server/resourcemanager/AdminService.java | 18 +-
.../server/resourcemanager/ResourceManager.java | 24 +-
.../RMStateVersionIncompatibleException.java | 2 +-
.../recovery/ZKRMStateStore.java | 5 +-
.../scheduler/MutableConfScheduler.java | 22 +-
.../scheduler/MutableConfigurationProvider.java | 36 ++-
.../scheduler/capacity/CapacityScheduler.java | 22 +-
.../conf/InMemoryConfigurationStore.java | 71 +++--
.../conf/LeveldbConfigurationStore.java | 168 +++++-----
.../conf/MutableCSConfigurationProvider.java | 148 +++++----
.../capacity/conf/YarnConfigurationStore.java | 132 ++++----
.../capacity/conf/ZKConfigurationStore.java | 235 ++++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 26 +-
.../conf/ConfigurationStoreBaseTest.java | 90 ++++++
.../conf/TestInMemoryConfigurationStore.java | 30 ++
.../TestMutableCSConfigurationProvider.java | 18 +-
.../conf/TestYarnConfigurationStore.java | 71 -----
.../capacity/conf/TestZKConfigurationStore.java | 312 +++++++++++++++++++
20 files changed, 1037 insertions(+), 422 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ea8652d..8809a7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -678,6 +678,7 @@ public class YarnConfiguration extends Configuration {
YARN_PREFIX + "scheduler.configuration.store.class";
public static final String MEMORY_CONFIGURATION_STORE = "memory";
public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+ public static final String ZK_CONFIGURATION_STORE = "zk";
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
@@ -689,9 +690,16 @@ public class YarnConfiguration extends Configuration {
public static final long
DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
- public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
- YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
- public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
+ public static final String RM_SCHEDCONF_MAX_LOGS =
+ YARN_PREFIX + "scheduler.configuration.store.max-logs";
+ public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
+ public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
+
+ /** Parent znode path under which ZKConfigurationStore will create znodes. */
+ public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX
+ + "scheduler.configuration.zk-store.parent-path";
+ public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH =
+ "/confstore";
public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5afec1b..4516edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3412,11 +3412,20 @@
<property>
<description>
- The max number of configuration change log entries kept in LevelDB config
+ The max number of configuration change log entries kept in config
store, when yarn.scheduler.configuration.store.class is configured to be
- "leveldb". Default is 1000.
+ "leveldb" or "zk". Default is 1000 for either.
</description>
- <name>yarn.scheduler.configuration.leveldb-store.max-logs</name>
+ <name>yarn.scheduler.configuration.store.max-logs</name>
<value>1000</value>
</property>
+
+ <property>
+ <description>
+ ZK root node path for configuration store when using zookeeper-based
+ configuration store.
+ </description>
+ <name>yarn.scheduler.configuration.zk-store.parent-path</name>
+ <value>/confstore</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index fd9e849..6c0a854 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -387,9 +387,7 @@ public class AdminService extends CompositeService implements
RefreshQueuesResponse response =
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
- ResourceScheduler scheduler = rm.getRMContext().getScheduler();
- if (scheduler instanceof MutableConfScheduler
- && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+ if (isSchedulerMutable()) {
throw new IOException("Scheduler configuration is mutable. " +
operation + " is not allowed in this scenario.");
}
@@ -413,6 +411,12 @@ public class AdminService extends CompositeService implements
}
}
+ private boolean isSchedulerMutable() {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ return (scheduler instanceof MutableConfScheduler
+ && ((MutableConfScheduler) scheduler).isConfigurationMutable());
+ }
+
@Override
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
throws YarnException, StandbyException {
@@ -721,6 +725,14 @@ public class AdminService extends CompositeService implements
void refreshAll() throws ServiceFailedException {
try {
checkAcls("refreshAll");
+ if (isSchedulerMutable()) {
+ try {
+ ((MutableConfScheduler) rm.getRMContext().getScheduler())
+ .getMutableConfProvider().reloadConfigurationFromStore();
+ } catch (Exception e) {
+ throw new IOException("Failed to refresh configuration:", e);
+ }
+ }
refreshQueues();
refreshNodes();
refreshSuperUserGroupsConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index fd95ee5..d9312f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -351,7 +351,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR,
YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED);
if (curatorEnabled) {
- this.zkManager = createAndStartZKManager(conf);
+ this.zkManager = getAndStartZKManager(conf);
elector = new CuratorBasedElectorService(this);
} else {
elector = new ActiveStandbyElectorBasedElectorService(this);
@@ -360,13 +360,16 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
/**
- * Create and ZooKeeper Curator manager.
+ * Get ZooKeeper Curator manager, creating and starting if not exists.
* @param config Configuration for the ZooKeeper curator.
- * @return New ZooKeeper Curator manager.
+ * @return ZooKeeper Curator manager.
* @throws IOException If it cannot create the manager.
*/
- public ZKCuratorManager createAndStartZKManager(Configuration config)
- throws IOException {
+ public synchronized ZKCuratorManager getAndStartZKManager(Configuration
+ config) throws IOException {
+ if (this.zkManager != null) {
+ return zkManager;
+ }
ZKCuratorManager manager = new ZKCuratorManager(config);
// Get authentication
@@ -386,15 +389,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
manager.start(authInfos);
- return manager;
- }
-
- /**
- * Get the ZooKeeper Curator manager.
- * @return ZooKeeper Curator manager.
- */
- public ZKCuratorManager getZKManager() {
- return this.zkManager;
+ this.zkManager = manager;
+ return zkManager;
}
public CuratorFramework getCurator() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
index 135868f..d5fce36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* This exception is thrown by ResourceManager if it's loading an incompatible
- * version of state from state store on recovery.
+ * version of storage on recovery.
*/
public class RMStateVersionIncompatibleException extends YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index ac67dcd..5bff77f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -327,10 +327,7 @@ public class ZKRMStateStore extends RMStateStore {
amrmTokenSecretManagerRoot =
getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT);
reservationRoot = getNodePath(zkRootNodePath, RESERVATION_SYSTEM_ROOT);
- zkManager = resourceManager.getZKManager();
- if (zkManager == null) {
- zkManager = resourceManager.createAndStartZKManager(conf);
- }
+ zkManager = resourceManager.getAndStartZKManager(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 313bf6a..6f677fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
-
-import java.io.IOException;
/**
* Interface for a scheduler that supports changing configuration at runtime.
@@ -31,16 +26,6 @@ import java.io.IOException;
public interface MutableConfScheduler extends ResourceScheduler {
/**
- * Update the scheduler's configuration.
- * @param user Caller of this update
- * @param confUpdate configuration update
- * @throws IOException if scheduler could not be reinitialized
- * @throws YarnException if reservation system could not be reinitialized
- */
- void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException, YarnException;
-
- /**
* Get the scheduler configuration.
* @return the scheduler configuration
*/
@@ -58,4 +43,11 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @return whether scheduler configuration is mutable or not.
*/
boolean isConfigurationMutable();
+
+ /**
+ * Get scheduler's configuration provider, so other classes can directly
+ * call mutation APIs on configuration provider.
+ * @return scheduler's configuration provider
+ */
+ MutableConfigurationProvider getMutableConfProvider();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9baf1ad..f8e8814 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,30 +19,40 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
-import java.io.IOException;
-
/**
* Interface for allowing changing scheduler configurations.
*/
public interface MutableConfigurationProvider {
/**
- * Apply transactions which were not committed.
- * @throws IOException if recovery fails
+ * Get the acl mutation policy for this configuration provider.
+ * @return The acl mutation policy.
+ */
+ ConfigurationMutationACLPolicy getAclMutationPolicy();
+
+ /**
+ * Called when a new ResourceManager is starting/becomes active. Ensures
+ * configuration is up-to-date.
+ * @throws Exception if configuration could not be refreshed from store
*/
- void recoverConf() throws IOException;
+ void reloadConfigurationFromStore() throws Exception;
/**
- * Update the scheduler configuration with the provided key value pairs.
- * @param user User issuing the request
- * @param confUpdate Key-value pairs for configurations to be updated.
- * @throws IOException if scheduler could not be reinitialized
- * @throws YarnException if reservation system could not be reinitialized
+ * Log user's requested configuration mutation, and applies it in-memory.
+ * @param user User who requested the change
+ * @param confUpdate User's requested configuration change
+ * @throws Exception if logging the mutation fails
*/
- void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
- confUpdate) throws IOException, YarnException;
+ void logAndApplyMutation(UserGroupInformation user, SchedConfUpdateInfo
+ confUpdate) throws Exception;
+ /**
+ * Confirm last logged mutation.
+ * @param isValid if the last logged mutation is applied to scheduler
+ * properly.
+ * @throws Exception if confirming mutation fails
+ */
+ void confirmPendingMutation(boolean isValid) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 51ee6a7..16b27c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -141,7 +141,6 @@ import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -393,9 +392,6 @@ public class CapacityScheduler extends
@Override
public void serviceStart() throws Exception {
startSchedulerThreads();
- if (this.csConfProvider instanceof MutableConfigurationProvider) {
- ((MutableConfigurationProvider) csConfProvider).recoverConf();
- }
super.serviceStart();
}
@@ -2619,19 +2615,15 @@ public class CapacityScheduler extends
}
@Override
- public void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
- if (isConfigurationMutable()) {
- ((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
- user, confUpdate);
- } else {
- throw new UnsupportedOperationException("Configured CS configuration " +
- "provider does not support updating configuration.");
- }
+ public boolean isConfigurationMutable() {
+ return csConfProvider instanceof MutableConfigurationProvider;
}
@Override
- public boolean isConfigurationMutable() {
- return csConfProvider instanceof MutableConfigurationProvider;
+ public MutableConfigurationProvider getMutableConfProvider() {
+ if (isConfigurationMutable()) {
+ return (MutableConfigurationProvider) csConfProvider;
+ }
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index c63734d..d69c236 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -19,8 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -28,48 +29,35 @@ import java.util.Map;
* A default implementation of {@link YarnConfigurationStore}. Doesn't offer
* persistent configuration storage, just stores the configuration in memory.
*/
-public class InMemoryConfigurationStore implements YarnConfigurationStore {
+public class InMemoryConfigurationStore extends YarnConfigurationStore {
private Configuration schedConf;
- private LinkedList<LogMutation> pendingMutations;
- private long pendingId;
+ private LogMutation pendingMutation;
@Override
- public void initialize(Configuration conf, Configuration schedConf) {
+ public void initialize(Configuration conf, Configuration schedConf,
+ RMContext rmContext) {
this.schedConf = schedConf;
- this.pendingMutations = new LinkedList<>();
- this.pendingId = 0;
}
@Override
- public synchronized long logMutation(LogMutation logMutation) {
- logMutation.setId(++pendingId);
- pendingMutations.add(logMutation);
- return pendingId;
+ public void logMutation(LogMutation logMutation) {
+ pendingMutation = logMutation;
}
@Override
- public synchronized boolean confirmMutation(long id, boolean isValid) {
- LogMutation mutation = pendingMutations.poll();
- // If confirmMutation is called out of order, discard mutations until id
- // is reached.
- while (mutation != null) {
- if (mutation.getId() == id) {
- if (isValid) {
- Map<String, String> mutations = mutation.getUpdates();
- for (Map.Entry<String, String> kv : mutations.entrySet()) {
- if (kv.getValue() == null) {
- schedConf.unset(kv.getKey());
- } else {
- schedConf.set(kv.getKey(), kv.getValue());
- }
- }
+ public void confirmMutation(boolean isValid) {
+ if (isValid) {
+ for (Map.Entry<String, String> kv : pendingMutation.getUpdates()
+ .entrySet()) {
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
}
- return true;
}
- mutation = pendingMutations.poll();
}
- return false;
+ pendingMutation = null;
}
@Override
@@ -78,13 +66,30 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
}
@Override
- public synchronized List<LogMutation> getPendingMutations() {
- return new LinkedList<>(pendingMutations);
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
}
@Override
- public List<LogMutation> getConfirmedConfHistory(long fromId) {
- // Unimplemented.
+ public Version getConfStoreVersion() throws Exception {
+ // Does nothing.
return null;
}
+
+ @Override
+ public void storeVersion() throws Exception {
+ // Does nothing.
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ // Does nothing.
+ return null;
+ }
+
+ @Override
+ public void checkVersion() {
+ // Does nothing. (Version is always compatible since it's in memory)
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 1280fab..1b0eb9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -26,6 +26,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.fusesource.leveldbjni.JniDBFactory;
import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
@@ -55,58 +59,32 @@ import static org.fusesource.leveldbjni.JniDBFactory.bytes;
/**
* A LevelDB implementation of {@link YarnConfigurationStore}.
*/
-public class LeveldbConfigurationStore implements YarnConfigurationStore {
+public class LeveldbConfigurationStore extends YarnConfigurationStore {
public static final Log LOG =
LogFactory.getLog(LeveldbConfigurationStore.class);
private static final String DB_NAME = "yarn-conf-store";
- private static final String LOG_PREFIX = "log.";
- private static final String LOG_COMMITTED_TXN = "committedTxn";
+ private static final String LOG_KEY = "log";
+ private static final String VERSION_KEY = "version";
private DB db;
- // Txnid for the last transaction logged to the store.
- private long txnId = 0;
- private long minTxn = 0;
private long maxLogs;
private Configuration conf;
- private LinkedList<LogMutation> pendingMutations = new LinkedList<>();
+ private LogMutation pendingMutation;
+ private static final Version CURRENT_VERSION_INFO = Version
+ .newInstance(0, 1);
private Timer compactionTimer;
private long compactionIntervalMsec;
@Override
- public void initialize(Configuration config, Configuration schedConf)
- throws IOException {
+ public void initialize(Configuration config, Configuration schedConf,
+ RMContext rmContext) throws IOException {
this.conf = config;
try {
this.db = initDatabase(schedConf);
- this.txnId = Long.parseLong(new String(db.get(bytes(LOG_COMMITTED_TXN)),
- StandardCharsets.UTF_8));
- DBIterator itr = db.iterator();
- itr.seek(bytes(LOG_PREFIX + txnId));
- // Seek to first uncommitted log
- itr.next();
- while (itr.hasNext()) {
- Map.Entry<byte[], byte[]> entry = itr.next();
- if (!new String(entry.getKey(), StandardCharsets.UTF_8)
- .startsWith(LOG_PREFIX)) {
- break;
- }
- pendingMutations.add(deserLogMutation(entry.getValue()));
- txnId++;
- }
- // Get the earliest txnId stored in logs
- itr.seekToFirst();
- if (itr.hasNext()) {
- Map.Entry<byte[], byte[]> entry = itr.next();
- byte[] key = entry.getKey();
- String logId = new String(key, StandardCharsets.UTF_8);
- if (logId.startsWith(LOG_PREFIX)) {
- minTxn = Long.parseLong(logId.substring(logId.indexOf('.') + 1));
- }
- }
this.maxLogs = config.getLong(
- YarnConfiguration.RM_SCHEDCONF_LEVELDB_MAX_LOGS,
+ YarnConfiguration.RM_SCHEDCONF_MAX_LOGS,
YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS);
this.compactionIntervalMsec = config.getLong(
YarnConfiguration.RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS,
@@ -127,33 +105,23 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
public int compare(byte[] key1, byte[] key2) {
String key1Str = new String(key1, StandardCharsets.UTF_8);
String key2Str = new String(key2, StandardCharsets.UTF_8);
- int key1Txn = Integer.MAX_VALUE;
- int key2Txn = Integer.MAX_VALUE;
- if (key1Str.startsWith(LOG_PREFIX)) {
- key1Txn = Integer.parseInt(key1Str.substring(
- key1Str.indexOf('.') + 1));
- }
- if (key2Str.startsWith(LOG_PREFIX)) {
- key2Txn = Integer.parseInt(key2Str.substring(
- key2Str.indexOf('.') + 1));
- }
- // TODO txnId could overflow, in theory
- if (key1Txn == Integer.MAX_VALUE && key2Txn == Integer.MAX_VALUE) {
- if (key1Str.equals(key2Str) && key1Str.equals(LOG_COMMITTED_TXN)) {
- return 0;
- } else if (key1Str.equals(LOG_COMMITTED_TXN)) {
- return -1;
- } else if (key2Str.equals(LOG_COMMITTED_TXN)) {
- return 1;
- }
- return key1Str.compareTo(key2Str);
+ if (key1Str.equals(key2Str)) {
+ return 0;
+ } else if (key1Str.equals(VERSION_KEY)) {
+ return -1;
+ } else if (key2Str.equals(VERSION_KEY)) {
+ return 1;
+ } else if (key1Str.equals(LOG_KEY)) {
+ return -1;
+ } else if (key2Str.equals(LOG_KEY)) {
+ return 1;
}
- return key1Txn - key2Txn;
+ return key1Str.compareTo(key2Str);
}
@Override
public String name() {
- return "logComparator";
+ return "keyComparator";
}
public byte[] findShortestSeparator(byte[] start, byte[] limit) {
@@ -164,6 +132,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
return key;
}
});
+
LOG.info("Using conf database at " + storeRoot);
File dbfile = new File(storeRoot.toString());
try {
@@ -179,7 +148,6 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
for (Map.Entry<String, String> kv : config) {
initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
}
- initBatch.put(bytes(LOG_COMMITTED_TXN), bytes("0"));
db.write(initBatch);
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
@@ -208,28 +176,22 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
@Override
- public synchronized long logMutation(LogMutation logMutation)
- throws IOException {
- logMutation.setId(++txnId);
- WriteBatch logBatch = db.createWriteBatch();
- logBatch.put(bytes(LOG_PREFIX + txnId), serLogMutation(logMutation));
- if (txnId - minTxn >= maxLogs) {
- logBatch.delete(bytes(LOG_PREFIX + minTxn));
- minTxn++;
+ public void logMutation(LogMutation logMutation) throws IOException {
+ LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.removeFirst();
}
- db.write(logBatch);
- pendingMutations.add(logMutation);
- return txnId;
+ db.put(bytes(LOG_KEY), serLogMutations(logs));
+ pendingMutation = logMutation;
}
@Override
- public synchronized boolean confirmMutation(long id, boolean isValid)
- throws IOException {
+ public void confirmMutation(boolean isValid) throws IOException {
WriteBatch updateBatch = db.createWriteBatch();
if (isValid) {
- LogMutation mutation = deserLogMutation(db.get(bytes(LOG_PREFIX + id)));
for (Map.Entry<String, String> changes :
- mutation.getUpdates().entrySet()) {
+ pendingMutation.getUpdates().entrySet()) {
if (changes.getValue() == null || changes.getValue().isEmpty()) {
updateBatch.delete(bytes(changes.getKey()));
} else {
@@ -237,28 +199,24 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
}
}
- updateBatch.put(bytes(LOG_COMMITTED_TXN), bytes(String.valueOf(id)));
db.write(updateBatch);
- // Assumes logMutation and confirmMutation are done in the same
- // synchronized method. For example,
- // {@link MutableCSConfigurationProvider#mutateConfiguration(
- // UserGroupInformation user, SchedConfUpdateInfo confUpdate)}
- pendingMutations.removeFirst();
- return true;
+ pendingMutation = null;
}
- private byte[] serLogMutation(LogMutation mutation) throws IOException {
+ private byte[] serLogMutations(LinkedList<LogMutation> mutations) throws
+ IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ObjectOutput oos = new ObjectOutputStream(baos)) {
- oos.writeObject(mutation);
+ oos.writeObject(mutations);
oos.flush();
return baos.toByteArray();
}
}
- private LogMutation deserLogMutation(byte[] mutation) throws IOException {
+ private LinkedList<LogMutation> deserLogMutations(byte[] mutations) throws
+ IOException {
try (ObjectInput input = new ObjectInputStream(
- new ByteArrayInputStream(mutation))) {
- return (LogMutation) input.readObject();
+ new ByteArrayInputStream(mutations))) {
+ return (LinkedList<LogMutation>) input.readObject();
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
@@ -267,7 +225,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
@Override
public synchronized Configuration retrieve() {
DBIterator itr = db.iterator();
- itr.seek(bytes(LOG_COMMITTED_TXN));
+ itr.seek(bytes(LOG_KEY));
Configuration config = new Configuration(false);
itr.next();
while (itr.hasNext()) {
@@ -279,11 +237,6 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
@Override
- public List<LogMutation> getPendingMutations() {
- return new LinkedList<>(pendingMutations);
- }
-
- @Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
return null; // unimplemented
}
@@ -299,6 +252,39 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
}
+ // TODO: following is taken from LeveldbRMStateStore
+ @Override
+ public Version getConfStoreVersion() throws Exception {
+ Version version = null;
+ try {
+ byte[] data = db.get(bytes(VERSION_KEY));
+ if (data != null) {
+ version = new VersionPBImpl(YarnServerCommonProtos.VersionProto
+ .parseFrom(data));
+ }
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ return version;
+ }
+
+ @Override
+ public void storeVersion() throws Exception {
+ String key = VERSION_KEY;
+ byte[] data = ((VersionPBImpl) CURRENT_VERSION_INFO).getProto()
+ .toByteArray();
+ try {
+ db.put(bytes(key), data);
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
private class CompactionTimerTask extends TimerTask {
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index d03b2e2..70d1840 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -18,20 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
@@ -56,6 +53,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
LogFactory.getLog(MutableCSConfigurationProvider.class);
private Configuration schedConf;
+ private Configuration oldConf;
private YarnConfigurationStore confStore;
private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
@@ -76,6 +74,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
this.confStore = new LeveldbConfigurationStore();
break;
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ this.confStore = new ZKConfigurationStore();
+ break;
default:
this.confStore = YarnConfigurationStoreFactory.getStore(config);
break;
@@ -89,7 +90,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
for (Map.Entry<String, String> kv : initialSchedConf) {
schedConf.set(kv.getKey(), kv.getValue());
}
- confStore.initialize(config, schedConf);
+ try {
+ confStore.initialize(config, schedConf, rmContext);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
// After initializing confStore, the store may already have an existing
// configuration. Use this one.
schedConf = confStore.retrieve();
@@ -98,6 +103,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
aclMutationPolicy.init(config, rmContext);
}
+ @VisibleForTesting
+ public YarnConfigurationStore getConfStore() {
+ return confStore;
+ }
+
@Override
public CapacitySchedulerConfiguration loadConfiguration(Configuration
configuration) throws IOException {
@@ -107,16 +117,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public synchronized void mutateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
- if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
- throw new AccessControlException("User is not admin of all modified" +
- " queues.");
- }
- Configuration oldConf = new Configuration(schedConf);
+ public ConfigurationMutationACLPolicy getAclMutationPolicy() {
+ return aclMutationPolicy;
+ }
+
+ @Override
+ public void logAndApplyMutation(UserGroupInformation user,
+ SchedConfUpdateInfo confUpdate) throws Exception {
+ oldConf = new Configuration(schedConf);
Map<String, String> kvUpdate = constructKeyValueConfUpdate(confUpdate);
LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
- long id = confStore.logMutation(log);
+ confStore.logMutation(log);
for (Map.Entry<String, String> kv : kvUpdate.entrySet()) {
if (kv.getValue() == null) {
schedConf.unset(kv.getKey());
@@ -124,47 +135,33 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
}
- try {
- rmContext.getRMAdminService().refreshQueues();
- } catch (IOException | YarnException e) {
+ }
+
+ @Override
+ public void confirmPendingMutation(boolean isValid) throws Exception {
+ confStore.confirmMutation(isValid);
+ if (!isValid) {
schedConf = oldConf;
- confStore.confirmMutation(id, false);
- throw e;
}
- confStore.confirmMutation(id, true);
}
@Override
- public void recoverConf() throws IOException {
- List<LogMutation> uncommittedLogs = confStore.getPendingMutations();
- Configuration oldConf = new Configuration(schedConf);
- for (LogMutation mutation : uncommittedLogs) {
- for (Map.Entry<String, String> kv : mutation.getUpdates().entrySet()) {
- if (kv.getValue() == null) {
- schedConf.unset(kv.getKey());
- } else {
- schedConf.set(kv.getKey(), kv.getValue());
- }
- }
- try {
- rmContext.getScheduler().reinitialize(schedConf, rmContext);
- } catch (IOException e) {
- schedConf = oldConf;
- confStore.confirmMutation(mutation.getId(), false);
- LOG.info("Configuration mutation " + mutation.getId()
- + " was rejected", e);
- continue;
- }
- confStore.confirmMutation(mutation.getId(), true);
- LOG.info("Configuration mutation " + mutation.getId()+ " was accepted");
- }
+ public void reloadConfigurationFromStore() throws Exception {
+ schedConf = confStore.retrieve();
+ }
+
+ private List<String> getSiblingQueues(String queuePath, Configuration conf) {
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
+ parentQueue + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES;
+ return new ArrayList<>(conf.getStringCollection(childQueuesKey));
}
private Map<String, String> constructKeyValueConfUpdate(
SchedConfUpdateInfo mutationInfo) throws IOException {
- CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
CapacitySchedulerConfiguration proposedConf =
- new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
+ new CapacitySchedulerConfiguration(schedConf, false);
Map<String, String> confUpdate = new HashMap<>();
for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
removeQueue(queueToRemove, proposedConf, confUpdate);
@@ -188,40 +185,35 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
if (queueToRemove == null) {
return;
} else {
- CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
String queueName = queueToRemove.substring(
queueToRemove.lastIndexOf('.') + 1);
- CSQueue queue = cs.getQueue(queueName);
- if (queue == null ||
- !queue.getQueuePath().equals(queueToRemove)) {
- throw new IOException("Queue " + queueToRemove + " not found");
- } else if (queueToRemove.lastIndexOf('.') == -1) {
+ if (queueToRemove.lastIndexOf('.') == -1) {
throw new IOException("Can't remove queue " + queueToRemove);
- }
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
- List<String> newSiblingQueues = new ArrayList<>();
- for (String siblingQueue : siblingQueues) {
- if (!siblingQueue.equals(queueName)) {
- newSiblingQueues.add(siblingQueue);
- }
- }
- proposedConf.setQueues(parentQueuePath, newSiblingQueues
- .toArray(new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX
- + parentQueuePath + CapacitySchedulerConfiguration.DOT
- + CapacitySchedulerConfiguration.QUEUES;
- if (newSiblingQueues.size() == 0) {
- confUpdate.put(queuesConfig, null);
} else {
- confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
- }
- for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
- .entrySet()) {
- proposedConf.unset(confRemove.getKey());
- confUpdate.put(confRemove.getKey(), null);
+ List<String> siblingQueues = getSiblingQueues(queueToRemove,
+ proposedConf);
+ if (!siblingQueues.contains(queueName)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ }
+ siblingQueues.remove(queueName);
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(
+ new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES;
+ if (siblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
}
}
}
@@ -232,13 +224,13 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
if (addInfo == null) {
return;
} else {
- CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
String queuePath = addInfo.getQueue();
String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (cs.getQueue(queueName) != null) {
- throw new IOException("Can't add existing queue " + queuePath);
- } else if (queuePath.lastIndexOf('.') == -1) {
+ if (queuePath.lastIndexOf('.') == -1) {
throw new IOException("Can't add invalid queue " + queuePath);
+ } else if (getSiblingQueues(queuePath, proposedConf).contains(
+ queueName)) {
+ throw new IOException("Can't add existing queue " + queuePath);
}
String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
String[] siblings = proposedConf.getQueues(parentQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 065c877..1356535 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateVersionIncompatibleException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import java.io.IOException;
@@ -39,36 +44,26 @@ import java.util.Map;
* {@code getPendingMutations}, and replay/confirm them via
* {@code confirmMutation} as in the normal case.
*/
-public interface YarnConfigurationStore {
+public abstract class YarnConfigurationStore {
+ public static final Log LOG =
+ LogFactory.getLog(YarnConfigurationStore.class);
/**
* LogMutation encapsulates the fields needed for configuration mutation
* audit logging and recovery.
*/
- class LogMutation implements Serializable {
+ static class LogMutation implements Serializable {
private Map<String, String> updates;
private String user;
- private long id;
/**
- * Create log mutation prior to logging.
+ * Create log mutation.
* @param updates key-value configuration updates
* @param user user who requested configuration change
*/
- public LogMutation(Map<String, String> updates, String user) {
- this(updates, user, 0);
- }
-
- /**
- * Create log mutation for recovery.
- * @param updates key-value configuration updates
- * @param user user who requested configuration change
- * @param id transaction id of configuration change
- */
- LogMutation(Map<String, String> updates, String user, long id) {
+ LogMutation(Map<String, String> updates, String user) {
this.updates = updates;
this.user = user;
- this.id = id;
}
/**
@@ -86,75 +81,92 @@ public interface YarnConfigurationStore {
public String getUser() {
return user;
}
-
- /**
- * Get transaction id of this configuration change.
- * @return transaction id
- */
- public long getId() {
- return id;
- }
-
- /**
- * Set transaction id of this configuration change.
- * @param id transaction id
- */
- public void setId(long id) {
- this.id = id;
- }
}
/**
- * Initialize the configuration store.
+ * Initialize the configuration store, with schedConf as the initial
+ * scheduler configuration. If a persisted store already exists, use the
+ * scheduler configuration stored there, and ignore schedConf.
* @param conf configuration to initialize store with
- * @param schedConf Initial key-value configuration to persist
+ * @param schedConf Initial key-value scheduler configuration to persist.
+ * @param rmContext RMContext for this configuration store
* @throws IOException if initialization fails
*/
- void initialize(Configuration conf, Configuration schedConf)
- throws IOException;
+ public abstract void initialize(Configuration conf, Configuration schedConf,
+ RMContext rmContext) throws Exception;
/**
- * Logs the configuration change to backing store. Generates an id associated
- * with this mutation, sets it in {@code logMutation}, and returns it.
+ * Logs the configuration change to backing store.
* @param logMutation configuration change to be persisted in write ahead log
- * @return id which configuration store associates with this mutation
* @throws IOException if logging fails
*/
- long logMutation(LogMutation logMutation) throws IOException;
+ public abstract void logMutation(LogMutation logMutation) throws Exception;
/**
* Should be called after {@code logMutation}. Gets the pending mutation
- * associated with {@code id} and marks the mutation as persisted (no longer
- * pending). If isValid is true, merge the mutation with the persisted
+ * last logged by {@code logMutation} and marks the mutation as persisted (no
+ * longer pending). If isValid is true, merge the mutation with the persisted
* configuration.
- *
- * If {@code confirmMutation} is called with ids in a different order than
- * was returned by {@code logMutation}, the result is implementation
- * dependent.
- * @param id id of mutation to be confirmed
- * @param isValid if true, update persisted configuration with mutation
- * associated with {@code id}.
- * @return true on success
- * @throws IOException if mutation confirmation fails
+ * @param isValid if true, update persisted configuration with pending
+ * mutation.
+ * @throws Exception if mutation confirmation fails
*/
- boolean confirmMutation(long id, boolean isValid) throws IOException;
+ public abstract void confirmMutation(boolean isValid) throws Exception;
/**
* Retrieve the persisted configuration.
* @return configuration as key-value
*/
- Configuration retrieve();
-
- /**
- * Get the list of pending mutations, in the order they were logged.
- * @return list of mutations
- */
- List<LogMutation> getPendingMutations();
+ public abstract Configuration retrieve();
/**
* Get a list of confirmed configuration mutations starting from a given id.
* @param fromId id from which to start getting mutations, inclusive
* @return list of configuration mutations
*/
- List<LogMutation> getConfirmedConfHistory(long fromId);
+ public abstract List<LogMutation> getConfirmedConfHistory(long fromId);
+
+ /**
+ * Get schema version of persisted conf store, for detecting compatibility
+ * issues when changing conf store schema.
+ * @return Schema version currently used by the persisted configuration store.
+ * @throws Exception On version fetch failure
+ */
+ protected abstract Version getConfStoreVersion() throws Exception;
+
+ /**
+ * Persist the hard-coded schema version to the conf store.
+ * @throws Exception On storage failure
+ */
+ protected abstract void storeVersion() throws Exception;
+
+ /**
+ * Get the hard-coded schema version, for comparison against the schema
+ * version currently persisted.
+ * @return Current hard-coded schema version
+ */
+ protected abstract Version getCurrentVersion();
+
+ public void checkVersion() throws Exception {
+ // TODO this was taken from RMStateStore. Should probably refactor
+ Version loadedVersion = getConfStoreVersion();
+ LOG.info("Loaded configuration store version info " + loadedVersion);
+ if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
+ return;
+ }
+ // if there is no version info, treat it as CURRENT_VERSION_INFO;
+ if (loadedVersion == null) {
+ loadedVersion = getCurrentVersion();
+ }
+ if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+ LOG.info("Storing configuration store version info "
+ + getCurrentVersion());
+ storeVersion();
+ } else {
+ throw new RMStateVersionIncompatibleException(
+ "Expecting configuration store version " + getCurrentVersion()
+ + ", but loading version " + loadedVersion);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
new file mode 100644
index 0000000..a0bba8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.ACL;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A Zookeeper-based implementation of {@link YarnConfigurationStore}.
+ */
+public class ZKConfigurationStore extends YarnConfigurationStore {
+
+ public static final Log LOG =
+ LogFactory.getLog(ZKConfigurationStore.class);
+
+ private long maxLogs;
+
+ @VisibleForTesting
+ protected static final Version CURRENT_VERSION_INFO = Version
+ .newInstance(0, 1);
+ private Configuration conf;
+ private LogMutation pendingMutation;
+
+ private String znodeParentPath;
+
+ private static final String ZK_VERSION_PATH = "VERSION";
+ private static final String LOGS_PATH = "LOGS";
+ private static final String CONF_STORE_PATH = "CONF_STORE";
+ private static final String FENCING_PATH = "FENCING";
+
+ private String zkVersionPath;
+ private String logsPath;
+ private String confStorePath;
+ private String fencingNodePath;
+
+ @VisibleForTesting
+ protected ZKCuratorManager zkManager;
+ private List<ACL> zkAcl;
+
+ @Override
+ public void initialize(Configuration config, Configuration schedConf,
+ RMContext rmContext) throws Exception {
+ this.conf = config;
+ this.maxLogs = conf.getLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS);
+ this.znodeParentPath =
+ conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH);
+ this.zkManager = rmContext.getResourceManager().getAndStartZKManager(conf);
+ this.zkAcl = ZKCuratorManager.getZKAcls(conf);
+
+ this.zkVersionPath = getNodePath(znodeParentPath, ZK_VERSION_PATH);
+ this.logsPath = getNodePath(znodeParentPath, LOGS_PATH);
+ this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH);
+ this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH);
+
+ zkManager.createRootDirRecursively(znodeParentPath);
+ zkManager.delete(fencingNodePath);
+
+ if (!zkManager.exists(logsPath)) {
+ zkManager.create(logsPath);
+ zkManager.setData(logsPath,
+ serializeObject(new LinkedList<LogMutation>()), -1);
+ }
+
+ if (!zkManager.exists(confStorePath)) {
+ zkManager.create(confStorePath);
+ HashMap<String, String> mapSchedConf = new HashMap<>();
+ for (Map.Entry<String, String> entry : schedConf) {
+ mapSchedConf.put(entry.getKey(), entry.getValue());
+ }
+ zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1);
+ }
+ }
+
+ @VisibleForTesting
+ protected LinkedList<LogMutation> getLogs() throws Exception {
+ return (LinkedList<LogMutation>)
+ deserializeObject(zkManager.getData(logsPath));
+ }
+
+ // TODO: following version-related code is taken from ZKRMStateStore
+ @Override
+ public Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
+ @Override
+ public Version getConfStoreVersion() throws Exception {
+ if (zkManager.exists(zkVersionPath)) {
+ byte[] data = zkManager.getData(zkVersionPath);
+ return new VersionPBImpl(YarnServerCommonProtos.VersionProto
+ .parseFrom(data));
+ }
+
+ return null;
+ }
+
+ @Override
+ public synchronized void storeVersion() throws Exception {
+ byte[] data =
+ ((VersionPBImpl) CURRENT_VERSION_INFO).getProto().toByteArray();
+
+ if (zkManager.exists(zkVersionPath)) {
+ zkManager.safeSetData(zkVersionPath, data, -1, zkAcl, fencingNodePath);
+ } else {
+ zkManager.safeCreate(zkVersionPath, data, zkAcl, CreateMode.PERSISTENT,
+ zkAcl, fencingNodePath);
+ }
+ }
+
+ @Override
+ public void logMutation(LogMutation logMutation) throws Exception {
+ byte[] storedLogs = zkManager.getData(logsPath);
+ LinkedList<LogMutation> logs = new LinkedList<>();
+ if (storedLogs != null) {
+ logs = (LinkedList<LogMutation>) deserializeObject(storedLogs);
+ }
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.remove(logs.removeFirst());
+ }
+ zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl,
+ fencingNodePath);
+ pendingMutation = logMutation;
+ }
+
+ @Override
+ public void confirmMutation(boolean isValid)
+ throws Exception {
+ if (isValid) {
+ Configuration storedConfigs = retrieve();
+ Map<String, String> mapConf = new HashMap<>();
+ for (Map.Entry<String, String> storedConf : storedConfigs) {
+ mapConf.put(storedConf.getKey(), storedConf.getValue());
+ }
+ for (Map.Entry<String, String> confChange :
+ pendingMutation.getUpdates().entrySet()) {
+ if (confChange.getValue() == null || confChange.getValue().isEmpty()) {
+ mapConf.remove(confChange.getKey());
+ } else {
+ mapConf.put(confChange.getKey(), confChange.getValue());
+ }
+ }
+ zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1,
+ zkAcl, fencingNodePath);
+ }
+ pendingMutation = null;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ byte[] serializedSchedConf;
+ try {
+ serializedSchedConf = zkManager.getData(confStorePath);
+ } catch (Exception e) {
+ LOG.error("Failed to retrieve configuration from zookeeper store", e);
+ return null;
+ }
+ try {
+ Map<String, String> map =
+ (HashMap<String, String>) deserializeObject(serializedSchedConf);
+ Configuration c = new Configuration();
+ for (Map.Entry<String, String> e : map.entrySet()) {
+ c.set(e.getKey(), e.getValue());
+ }
+ return c;
+ } catch (Exception e) {
+ LOG.error("Exception while deserializing scheduler configuration " +
+ "from store", e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ return null; // unimplemented
+ }
+
+ private static String getNodePath(String root, String nodeName) {
+ return ZKCuratorManager.getNodePath(root, nodeName);
+ }
+
+ private static byte[] serializeObject(Object o) throws Exception {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);) {
+ oos.writeObject(o);
+ oos.flush();
+ baos.flush();
+ return baos.toByteArray();
+ }
+ }
+
+ private static Object deserializeObject(byte[] bytes) throws Exception {
+ try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ ObjectInputStream ois = new ObjectInputStream(bais);) {
+ return ois.readObject();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 1da4e65..d264c10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -2464,7 +2465,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response updateSchedulerConfiguration(SchedConfUpdateInfo
+ public synchronized Response updateSchedulerConfiguration(SchedConfUpdateInfo
mutationInfo, @Context HttpServletRequest hsr)
throws AuthorizationException, InterruptedException {
init();
@@ -2479,17 +2480,32 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
ResourceScheduler scheduler = rm.getResourceScheduler();
- if (scheduler instanceof MutableConfScheduler) {
+ if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler)
+ scheduler).isConfigurationMutable()) {
try {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws IOException, YarnException {
- ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI,
- mutationInfo);
+ public Void run() throws Exception {
+ MutableConfigurationProvider provider = ((MutableConfScheduler)
+ scheduler).getMutableConfProvider();
+ if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI,
+ mutationInfo)) {
+ throw new org.apache.hadoop.security.AccessControlException("User"
+ + " is not admin of all modified queues.");
+ }
+ provider.logAndApplyMutation(callerUGI, mutationInfo);
+ try {
+ rm.getRMContext().getRMAdminService().refreshQueues();
+ } catch (IOException | YarnException e) {
+ provider.confirmPendingMutation(false);
+ throw e;
+ }
+ provider.confirmPendingMutation(true);
return null;
}
});
} catch (IOException e) {
+ LOG.error("Exception thrown when modifying configuration.", e);
return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
new file mode 100644
index 0000000..bbe9570
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Base class for {@link YarnConfigurationStore} implementations.
+ */
+public abstract class ConfigurationStoreBaseTest {
+
+ protected YarnConfigurationStore confStore = createConfStore();
+
+ protected abstract YarnConfigurationStore createConfStore();
+
+ protected Configuration conf;
+ protected Configuration schedConf;
+ protected RMContext rmContext;
+
+ protected static final String TEST_USER = "testUser";
+
+ @Before
+ public void setUp() throws Exception {
+ this.conf = new Configuration();
+ this.schedConf = new Configuration(false);
+ }
+
+ @Test
+ public void testConfigurationUpdate() throws Exception {
+ schedConf.set("key1", "val1");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ YarnConfigurationStore.LogMutation mutation1 =
+ new YarnConfigurationStore.LogMutation(update1, TEST_USER);
+ confStore.logMutation(mutation1);
+ confStore.confirmMutation(true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ YarnConfigurationStore.LogMutation mutation2 =
+ new YarnConfigurationStore.LogMutation(update2, TEST_USER);
+ confStore.logMutation(mutation2);
+ confStore.confirmMutation(false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ }
+
+ @Test
+ public void testNullConfigurationUpdate() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key", null);
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+ confStore.confirmMutation(true);
+ assertNull(confStore.retrieve().get("key"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
new file mode 100644
index 0000000..c40d16a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+/**
+ * Tests {@link InMemoryConfigurationStore}.
+ */
+public class TestInMemoryConfigurationStore extends ConfigurationStoreBaseTest {
+
+ @Override
+ protected YarnConfigurationStore createConfStore() {
+ return new InMemoryConfigurationStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 635a184..9b080cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -30,14 +29,11 @@ import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -82,25 +78,21 @@ public class TestMutableCSConfigurationProvider {
}
@Test
- public void testInMemoryBackedProvider() throws IOException, YarnException {
+ public void testInMemoryBackedProvider() throws Exception {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
- doNothing().when(adminService).refreshQueues();
- confProvider.mutateConfiguration(TEST_USER, goodUpdate);
+ confProvider.logAndApplyMutation(TEST_USER, goodUpdate);
+ confProvider.confirmPendingMutation(true);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
- doThrow(new IOException()).when(adminService).refreshQueues();
- try {
- confProvider.mutateConfiguration(TEST_USER, badUpdate);
- } catch (IOException e) {
- // Expected exception.
- }
+ confProvider.logAndApplyMutation(TEST_USER, badUpdate);
+ confProvider.confirmPendingMutation(false);
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff39c0de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
deleted file mode 100644
index 631ce65..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-public class TestYarnConfigurationStore {
-
- private YarnConfigurationStore confStore;
- private Configuration schedConf;
-
- private static final String testUser = "testUser";
-
- @Before
- public void setUp() {
- schedConf = new Configuration(false);
- schedConf.set("key1", "val1");
- }
-
- @Test
- public void testInMemoryConfigurationStore() throws IOException {
- confStore = new InMemoryConfigurationStore();
- confStore.initialize(new Configuration(), schedConf);
- assertEquals("val1", confStore.retrieve().get("key1"));
-
- Map<String, String> update1 = new HashMap<>();
- update1.put("keyUpdate1", "valUpdate1");
- LogMutation mutation1 = new LogMutation(update1, testUser);
- long id = confStore.logMutation(mutation1);
- assertEquals(1, confStore.getPendingMutations().size());
- confStore.confirmMutation(id, true);
- assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
- assertEquals(0, confStore.getPendingMutations().size());
-
- Map<String, String> update2 = new HashMap<>();
- update2.put("keyUpdate2", "valUpdate2");
- LogMutation mutation2 = new LogMutation(update2, testUser);
- id = confStore.logMutation(mutation2);
- assertEquals(1, confStore.getPendingMutations().size());
- confStore.confirmMutation(id, false);
- assertNull("Configuration should not be updated",
- confStore.retrieve().get("keyUpdate2"));
- assertEquals(0, confStore.getPendingMutations().size());
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-7024: Fix issues on recovery
in LevelDB store. Contributed by Jonathan Hung
Posted by su...@apache.org.
YARN-7024: Fix issues on recovery in LevelDB store. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d8abd84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d8abd84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d8abd84
Branch: refs/heads/YARN-5881
Commit: 4d8abd84f40a6124e7b05b34c14b4035309623ef
Parents: 4c8b208
Author: Xuan <xg...@apache.org>
Authored: Wed Aug 23 11:11:41 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/conf/InMemoryConfigurationStore.java | 2 +-
.../scheduler/capacity/conf/LeveldbConfigurationStore.java | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d8abd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index b97be1b..c63734d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -79,7 +79,7 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
@Override
public synchronized List<LogMutation> getPendingMutations() {
- return pendingMutations;
+ return new LinkedList<>(pendingMutations);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d8abd84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 1534685..1280fab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -65,6 +65,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
private static final String LOG_COMMITTED_TXN = "committedTxn";
private DB db;
+ // Txnid for the last transaction logged to the store.
private long txnId = 0;
private long minTxn = 0;
private long maxLogs;
@@ -92,6 +93,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
break;
}
pendingMutations.add(deserLogMutation(entry.getValue()));
+ txnId++;
}
// Get the earliest txnId stored in logs
itr.seekToFirst();
@@ -278,7 +280,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
@Override
public List<LogMutation> getPendingMutations() {
- return pendingMutations;
+ return new LinkedList<>(pendingMutations);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-5948. Implement
MutableConfigurationManager for handling storage into configuration store
Posted by su...@apache.org.
YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef686635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef686635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef686635
Branch: refs/heads/YARN-5881
Commit: ef686635c1aa0cd4bd52ca212a95769f2532aa1a
Parents: e3579a8
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Wed Mar 1 16:03:01 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 ++
.../src/main/resources/yarn-default.xml | 12 +++
.../scheduler/MutableConfigurationProvider.java | 35 ++++++++
.../scheduler/capacity/CapacityScheduler.java | 14 ++-
.../CapacitySchedulerConfiguration.java | 3 +
.../capacity/conf/CSConfigurationProvider.java | 3 +-
.../conf/MutableCSConfigurationProvider.java | 94 ++++++++++++++++++++
.../conf/YarnConfigurationStoreFactory.java | 46 ++++++++++
.../TestMutableCSConfigurationProvider.java | 83 +++++++++++++++++
9 files changed, 291 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d99941d..92b9697 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -674,6 +674,12 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
+ YARN_PREFIX + "scheduler.configuration.store.class";
+ public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ public static final String DEFAULT_CONFIGURATION_STORE =
+ MEMORY_CONFIGURATION_STORE;
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2bc2330..73f4717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3367,4 +3367,16 @@
</description>
</property>
+ <property>
+ <description>
+ The type of configuration store to use for storing scheduler
+ configurations, if using a mutable configuration provider.
+ Keywords such as "memory" map to certain configuration store
+ implementations. If keyword is not found, try to load this
+ value as a class.
+ </description>
+ <name>yarn.scheduler.configuration.store.class</name>
+ <value>memory</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
new file mode 100644
index 0000000..da30a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.Map;
+
+/**
+ * Interface for allowing changing scheduler configurations.
+ */
+public interface MutableConfigurationProvider {
+
+ /**
+ * Update the scheduler configuration with the provided key value pairs.
+ * @param user User issuing the request
+ * @param confUpdate Key-value pairs for configurations to be updated.
+ */
+ void mutateConfiguration(String user, Map<String, String> confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e74cbe1..11f2f6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.MutableCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -295,10 +296,15 @@ public class CapacityScheduler extends
String confProviderStr = configuration.get(
CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
- if (confProviderStr.equals(
- CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
- this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
- } else {
+ switch (confProviderStr) {
+ case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER:
+ this.csConfProvider =
+ new FileBasedCSConfigurationProvider(rmContext);
+ break;
+ case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER:
+ this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
+ break;
+ default:
throw new IOException("Invalid CS configuration provider: " +
confProviderStr);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3821e24..40cb893 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -323,6 +323,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String FILE_CS_CONF_PROVIDER = "file";
@Private
+ public static final String STORE_CS_CONF_PROVIDER = "store";
+
+ @Private
public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
index c9984ac..0d2c8bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -32,8 +32,9 @@ public interface CSConfigurationProvider {
/**
* Initialize the configuration provider with given conf.
* @param conf configuration to initialize with
+ * @throws IOException if initialization fails due to misconfiguration
*/
- void init(Configuration conf);
+ void init(Configuration conf) throws IOException;
/**
* Loads capacity scheduler configuration object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
new file mode 100644
index 0000000..267ab6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * CS configuration provider which implements
+ * {@link MutableConfigurationProvider} for modifying capacity scheduler
+ * configuration.
+ */
+public class MutableCSConfigurationProvider implements CSConfigurationProvider,
+ MutableConfigurationProvider {
+
+ private Configuration schedConf;
+ private YarnConfigurationStore confStore;
+ private RMContext rmContext;
+ private Configuration conf;
+
+ public MutableCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration config) throws IOException {
+ String store = config.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ switch (store) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ this.confStore = new InMemoryConfigurationStore();
+ break;
+ default:
+ this.confStore = YarnConfigurationStoreFactory.getStore(config);
+ break;
+ }
+ Configuration initialSchedConf = new Configuration(false);
+ initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
+ this.schedConf = initialSchedConf;
+ confStore.initialize(config, initialSchedConf);
+ this.conf = config;
+ }
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration
+ configuration) throws IOException {
+ Configuration loadedConf = new Configuration(configuration);
+ loadedConf.addResource(schedConf);
+ return new CapacitySchedulerConfiguration(loadedConf, false);
+ }
+
+ @Override
+ public void mutateConfiguration(String user,
+ Map<String, String> confUpdate) {
+ Configuration oldConf = new Configuration(schedConf);
+ LogMutation log = new LogMutation(confUpdate, user);
+ long id = confStore.logMutation(log);
+ for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ try {
+ rmContext.getScheduler().reinitialize(conf, rmContext);
+ } catch (IOException e) {
+ schedConf = oldConf;
+ confStore.confirmMutation(id, false);
+ return;
+ }
+ confStore.confirmMutation(id, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
new file mode 100644
index 0000000..60249c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of {@link YarnConfigurationStore}.
+ */
+public final class YarnConfigurationStoreFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ YarnConfigurationStoreFactory.class);
+
+ private YarnConfigurationStoreFactory() {
+ // Unused.
+ }
+
+ public static YarnConfigurationStore getStore(Configuration conf) {
+ Class<? extends YarnConfigurationStore> storeClass =
+ conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ InMemoryConfigurationStore.class, YarnConfigurationStore.class);
+ LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef686635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
new file mode 100644
index 0000000..3f103b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests {@link MutableCSConfigurationProvider}.
+ */
+public class TestMutableCSConfigurationProvider {
+
+ private MutableCSConfigurationProvider confProvider;
+ private RMContext rmContext;
+ private Map<String, String> goodUpdate;
+ private Map<String, String> badUpdate;
+ private CapacityScheduler cs;
+
+ private static final String TEST_USER = "testUser";
+
+ @Before
+ public void setUp() {
+ cs = mock(CapacityScheduler.class);
+ rmContext = mock(RMContext.class);
+ when(rmContext.getScheduler()).thenReturn(cs);
+ confProvider = new MutableCSConfigurationProvider(rmContext);
+ goodUpdate = new HashMap<>();
+ goodUpdate.put("goodKey", "goodVal");
+ badUpdate = new HashMap<>();
+ badUpdate.put("badKey", "badVal");
+ }
+
+ @Test
+ public void testInMemoryBackedProvider() throws IOException {
+ Configuration conf = new Configuration();
+ confProvider.init(conf);
+ assertNull(confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ doNothing().when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, goodUpdate);
+ assertEquals("goodVal", confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-5947: Create
LeveldbConfigurationStore class using Leveldb as backing store. Contributed
by Jonathan Hung
Posted by su...@apache.org.
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74ba6ffa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74ba6ffa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74ba6ffa
Branch: refs/heads/YARN-5881
Commit: 74ba6ffa0b601badc68366685dcf993a0844520a
Parents: 04f9e80
Author: Xuan <xg...@apache.org>
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Mon Oct 9 11:11:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../src/main/resources/yarn-default.xml | 29 ++
.../scheduler/MutableConfigurationProvider.java | 6 +
.../scheduler/capacity/CapacityScheduler.java | 3 +
.../conf/LeveldbConfigurationStore.java | 314 +++++++++++++++++++
.../conf/MutableCSConfigurationProvider.java | 38 ++-
.../capacity/conf/YarnConfigurationStore.java | 14 +-
.../conf/TestYarnConfigurationStore.java | 3 +-
8 files changed, 414 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
YARN_PREFIX + "scheduler.configuration.store.class";
public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
+ public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+ + "scheduler.configuration.leveldb-store.path";
+
+ public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+ YARN_PREFIX
+ + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+ public static final long
+ DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+ public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+ YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+ public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
</property>
+ <property>
+ <description>
+ The storage path for LevelDB implementation of configuration store,
+ when yarn.scheduler.configuration.store.class is configured to be
+ "leveldb".
+ </description>
+ <name>yarn.scheduler.configuration.leveldb-store.path</name>
+ <value>${hadoop.tmp.dir}/yarn/system/confstore</value>
+ </property>
+
+ <property>
+ <description>
+ The compaction interval for LevelDB configuration store in secs,
+ when yarn.scheduler.configuration.store.class is configured to be
+ "leveldb". Default is one day.
+ </description>
+ <name>yarn.scheduler.configuration.leveldb-store.compaction-interval-secs</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <description>
+ The max number of configuration change log entries kept in LevelDB config
+ store, when yarn.scheduler.configuration.store.class is configured to be
+ "leveldb". Default is 1000.
+ </description>
+ <name>yarn.scheduler.configuration.leveldb-store.max-logs</name>
+ <value>1000</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 86be7c3..1f13467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -29,6 +29,12 @@ import java.io.IOException;
public interface MutableConfigurationProvider {
/**
+ * Apply transactions which were not committed.
+ * @throws IOException if recovery fails
+ */
+ void recoverConf() throws IOException;
+
+ /**
* Update the scheduler configuration with the provided key value pairs.
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index da395b7..6d2de7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -393,6 +393,9 @@ public class CapacityScheduler extends
@Override
public void serviceStart() throws Exception {
startSchedulerThreads();
+ if (this.csConfProvider instanceof MutableConfigurationProvider) {
+ ((MutableConfigurationProvider) csConfProvider).recoverConf();
+ }
super.serviceStart();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
new file mode 100644
index 0000000..1534685
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.fusesource.leveldbjni.internal.NativeDB;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBComparator;
+import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.DBIterator;
+import org.iq80.leveldb.Options;
+import org.iq80.leveldb.WriteBatch;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+
+/**
+ * A LevelDB implementation of {@link YarnConfigurationStore}.
+ */
+public class LeveldbConfigurationStore implements YarnConfigurationStore {
+
+ public static final Log LOG =
+ LogFactory.getLog(LeveldbConfigurationStore.class);
+
+ private static final String DB_NAME = "yarn-conf-store";
+ private static final String LOG_PREFIX = "log.";
+ private static final String LOG_COMMITTED_TXN = "committedTxn";
+
+ private DB db;
+ private long txnId = 0;
+ private long minTxn = 0;
+ private long maxLogs;
+ private Configuration conf;
+ private LinkedList<LogMutation> pendingMutations = new LinkedList<>();
+ private Timer compactionTimer;
+ private long compactionIntervalMsec;
+
+ @Override
+ public void initialize(Configuration config, Configuration schedConf)
+ throws IOException {
+ this.conf = config;
+ try {
+ this.db = initDatabase(schedConf);
+ this.txnId = Long.parseLong(new String(db.get(bytes(LOG_COMMITTED_TXN)),
+ StandardCharsets.UTF_8));
+ DBIterator itr = db.iterator();
+ itr.seek(bytes(LOG_PREFIX + txnId));
+ // Seek to first uncommitted log
+ itr.next();
+ while (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ if (!new String(entry.getKey(), StandardCharsets.UTF_8)
+ .startsWith(LOG_PREFIX)) {
+ break;
+ }
+ pendingMutations.add(deserLogMutation(entry.getValue()));
+ }
+ // Get the earliest txnId stored in logs
+ itr.seekToFirst();
+ if (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ byte[] key = entry.getKey();
+ String logId = new String(key, StandardCharsets.UTF_8);
+ if (logId.startsWith(LOG_PREFIX)) {
+ minTxn = Long.parseLong(logId.substring(logId.indexOf('.') + 1));
+ }
+ }
+ this.maxLogs = config.getLong(
+ YarnConfiguration.RM_SCHEDCONF_LEVELDB_MAX_LOGS,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS);
+ this.compactionIntervalMsec = config.getLong(
+ YarnConfiguration.RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS,
+ YarnConfiguration
+ .DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS) * 1000;
+ startCompactionTimer();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private DB initDatabase(Configuration config) throws Exception {
+ Path storeRoot = createStorageDir();
+ Options options = new Options();
+ options.createIfMissing(false);
+ options.comparator(new DBComparator() {
+ @Override
+ public int compare(byte[] key1, byte[] key2) {
+ String key1Str = new String(key1, StandardCharsets.UTF_8);
+ String key2Str = new String(key2, StandardCharsets.UTF_8);
+ int key1Txn = Integer.MAX_VALUE;
+ int key2Txn = Integer.MAX_VALUE;
+ if (key1Str.startsWith(LOG_PREFIX)) {
+ key1Txn = Integer.parseInt(key1Str.substring(
+ key1Str.indexOf('.') + 1));
+ }
+ if (key2Str.startsWith(LOG_PREFIX)) {
+ key2Txn = Integer.parseInt(key2Str.substring(
+ key2Str.indexOf('.') + 1));
+ }
+ // TODO txnId could overflow, in theory
+ if (key1Txn == Integer.MAX_VALUE && key2Txn == Integer.MAX_VALUE) {
+ if (key1Str.equals(key2Str) && key1Str.equals(LOG_COMMITTED_TXN)) {
+ return 0;
+ } else if (key1Str.equals(LOG_COMMITTED_TXN)) {
+ return -1;
+ } else if (key2Str.equals(LOG_COMMITTED_TXN)) {
+ return 1;
+ }
+ return key1Str.compareTo(key2Str);
+ }
+ return key1Txn - key2Txn;
+ }
+
+ @Override
+ public String name() {
+ return "logComparator";
+ }
+
+ public byte[] findShortestSeparator(byte[] start, byte[] limit) {
+ return start;
+ }
+
+ public byte[] findShortSuccessor(byte[] key) {
+ return key;
+ }
+ });
+ LOG.info("Using conf database at " + storeRoot);
+ File dbfile = new File(storeRoot.toString());
+ try {
+ db = JniDBFactory.factory.open(dbfile, options);
+ } catch (NativeDB.DBException e) {
+ if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
+ LOG.info("Creating conf database at " + dbfile);
+ options.createIfMissing(true);
+ try {
+ db = JniDBFactory.factory.open(dbfile, options);
+ // Write the initial scheduler configuration
+ WriteBatch initBatch = db.createWriteBatch();
+ for (Map.Entry<String, String> kv : config) {
+ initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
+ }
+ initBatch.put(bytes(LOG_COMMITTED_TXN), bytes("0"));
+ db.write(initBatch);
+ } catch (DBException dbErr) {
+ throw new IOException(dbErr.getMessage(), dbErr);
+ }
+ } else {
+ throw e;
+ }
+ }
+ return db;
+ }
+
+ private Path createStorageDir() throws IOException {
+ Path root = getStorageDir();
+ FileSystem fs = FileSystem.getLocal(conf);
+ fs.mkdirs(root, new FsPermission((short) 0700));
+ return root;
+ }
+
+ private Path getStorageDir() throws IOException {
+ String storePath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
+ if (storePath == null) {
+ throw new IOException("No store location directory configured in " +
+ YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
+ }
+ return new Path(storePath, DB_NAME);
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation)
+ throws IOException {
+ logMutation.setId(++txnId);
+ WriteBatch logBatch = db.createWriteBatch();
+ logBatch.put(bytes(LOG_PREFIX + txnId), serLogMutation(logMutation));
+ if (txnId - minTxn >= maxLogs) {
+ logBatch.delete(bytes(LOG_PREFIX + minTxn));
+ minTxn++;
+ }
+ db.write(logBatch);
+ pendingMutations.add(logMutation);
+ return txnId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid)
+ throws IOException {
+ WriteBatch updateBatch = db.createWriteBatch();
+ if (isValid) {
+ LogMutation mutation = deserLogMutation(db.get(bytes(LOG_PREFIX + id)));
+ for (Map.Entry<String, String> changes :
+ mutation.getUpdates().entrySet()) {
+ if (changes.getValue() == null || changes.getValue().isEmpty()) {
+ updateBatch.delete(bytes(changes.getKey()));
+ } else {
+ updateBatch.put(bytes(changes.getKey()), bytes(changes.getValue()));
+ }
+ }
+ }
+ updateBatch.put(bytes(LOG_COMMITTED_TXN), bytes(String.valueOf(id)));
+ db.write(updateBatch);
+ // Assumes logMutation and confirmMutation are done in the same
+ // synchronized method. For example,
+ // {@link MutableCSConfigurationProvider#mutateConfiguration(
+ // UserGroupInformation user, SchedConfUpdateInfo confUpdate)}
+ pendingMutations.removeFirst();
+ return true;
+ }
+
+ private byte[] serLogMutation(LogMutation mutation) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ObjectOutput oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(mutation);
+ oos.flush();
+ return baos.toByteArray();
+ }
+ }
+ private LogMutation deserLogMutation(byte[] mutation) throws IOException {
+ try (ObjectInput input = new ObjectInputStream(
+ new ByteArrayInputStream(mutation))) {
+ return (LogMutation) input.readObject();
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ DBIterator itr = db.iterator();
+ itr.seek(bytes(LOG_COMMITTED_TXN));
+ Configuration config = new Configuration(false);
+ itr.next();
+ while (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ config.set(new String(entry.getKey(), StandardCharsets.UTF_8),
+ new String(entry.getValue(), StandardCharsets.UTF_8));
+ }
+ return config;
+ }
+
+ @Override
+ public List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ return null; // unimplemented
+ }
+
+ // TODO below was taken from LeveldbRMStateStore, it can probably be
+ // refactored
+ private void startCompactionTimer() {
+ if (compactionIntervalMsec > 0) {
+ compactionTimer = new Timer(
+ this.getClass().getSimpleName() + " compaction timer", true);
+ compactionTimer.schedule(new CompactionTimerTask(),
+ compactionIntervalMsec, compactionIntervalMsec);
+ }
+ }
+
+ private class CompactionTimerTask extends TimerTask {
+ @Override
+ public void run() {
+ long start = Time.monotonicNow();
+ LOG.info("Starting full compaction cycle");
+ try {
+ db.compactRange(null, null);
+ } catch (DBException e) {
+ LOG.error("Error compacting database", e);
+ }
+ long duration = Time.monotonicNow() - start;
+ LOG.info("Full compaction cycle completed in " + duration + " msec");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 670c0f9..9ccc146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import com.google.common.base.Joiner;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -49,6 +51,9 @@ import java.util.Map;
public class MutableCSConfigurationProvider implements CSConfigurationProvider,
MutableConfigurationProvider {
+ public static final Log LOG =
+ LogFactory.getLog(MutableCSConfigurationProvider.class);
+
private Configuration schedConf;
private YarnConfigurationStore confStore;
private ConfigurationMutationACLPolicy aclMutationPolicy;
@@ -68,6 +73,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
this.confStore = new InMemoryConfigurationStore();
break;
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ this.confStore = new LeveldbConfigurationStore();
+ break;
default:
this.confStore = YarnConfigurationStoreFactory.getStore(config);
break;
@@ -82,6 +90,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
confStore.initialize(config, schedConf);
+ // After initializing confStore, the store may already have an existing
+ // configuration. Use this one.
+ schedConf = confStore.retrieve();
this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
.getPolicy(config);
aclMutationPolicy.init(config, rmContext);
@@ -97,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public void mutateConfiguration(UserGroupInformation user,
+ public synchronized void mutateConfiguration(UserGroupInformation user,
SchedConfUpdateInfo confUpdate) throws IOException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
@@ -124,6 +135,31 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
confStore.confirmMutation(id, true);
}
+ @Override
+ public void recoverConf() throws IOException {
+ List<LogMutation> uncommittedLogs = confStore.getPendingMutations();
+ Configuration oldConf = new Configuration(schedConf);
+ for (LogMutation mutation : uncommittedLogs) {
+ for (Map.Entry<String, String> kv : mutation.getUpdates().entrySet()) {
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ try {
+ rmContext.getScheduler().reinitialize(conf, rmContext);
+ } catch (IOException e) {
+ schedConf = oldConf;
+ confStore.confirmMutation(mutation.getId(), false);
+ LOG.info("Configuration mutation " + mutation.getId()
+ + " was rejected", e);
+ continue;
+ }
+ confStore.confirmMutation(mutation.getId(), true);
+ LOG.info("Configuration mutation " + mutation.getId()+ " was accepted");
+ }
+ }
private Map<String, String> constructKeyValueConfUpdate(
SchedConfUpdateInfo mutationInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 22c0ef8..065c877 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import java.io.IOException;
+import java.io.Serializable;
import java.util.List;
import java.util.Map;
@@ -43,7 +45,7 @@ public interface YarnConfigurationStore {
* LogMutation encapsulates the fields needed for configuration mutation
* audit logging and recovery.
*/
- class LogMutation {
+ class LogMutation implements Serializable {
private Map<String, String> updates;
private String user;
private long id;
@@ -106,16 +108,19 @@ public interface YarnConfigurationStore {
* Initialize the configuration store.
* @param conf configuration to initialize store with
* @param schedConf Initial key-value configuration to persist
+ * @throws IOException if initialization fails
*/
- void initialize(Configuration conf, Configuration schedConf);
+ void initialize(Configuration conf, Configuration schedConf)
+ throws IOException;
/**
* Logs the configuration change to backing store. Generates an id associated
* with this mutation, sets it in {@code logMutation}, and returns it.
* @param logMutation configuration change to be persisted in write ahead log
* @return id which configuration store associates with this mutation
+ * @throws IOException if logging fails
*/
- long logMutation(LogMutation logMutation);
+ long logMutation(LogMutation logMutation) throws IOException;
/**
* Should be called after {@code logMutation}. Gets the pending mutation
@@ -130,8 +135,9 @@ public interface YarnConfigurationStore {
* @param isValid if true, update persisted configuration with mutation
* associated with {@code id}.
* @return true on success
+ * @throws IOException if mutation confirmation fails
*/
- boolean confirmMutation(long id, boolean isValid);
+ boolean confirmMutation(long id, boolean isValid) throws IOException;
/**
* Retrieve the persisted configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
index dff4e77..631ce65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -43,7 +44,7 @@ public class TestYarnConfigurationStore {
}
@Test
- public void testInMemoryConfigurationStore() {
+ public void testInMemoryConfigurationStore() throws IOException {
confStore = new InMemoryConfigurationStore();
confStore.initialize(new Configuration(), schedConf);
assertEquals("val1", confStore.retrieve().get("key1"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: MAPREDUCE-6951. Improve exception
message when mapreduce.jobhistory.webapp.address is in wrong format.
Contributed by Prabhu Joseph.
Posted by su...@apache.org.
MAPREDUCE-6951. Improve exception message when mapreduce.jobhistory.webapp.address is in wrong format. Contributed by Prabhu Joseph.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/639f98cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/639f98cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/639f98cc
Branch: refs/heads/YARN-5881
Commit: 639f98cc9dc9fca034d0d7c26f30d5c588b9d7b8
Parents: d8d37b6
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Oct 11 14:27:38 2017 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Oct 11 14:27:38 2017 +0530
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java | 13 ++++++++++---
.../apache/hadoop/mapreduce/v2/util/TestMRApps.java | 9 +++++++++
2 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/639f98cc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
index 951c9d5..6f2e21f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
+import java.util.NoSuchElementException;
import java.util.Iterator;
import static org.apache.hadoop.http.HttpConfig.Policy;
@@ -126,9 +127,15 @@ public class MRWebAppUtil {
throws UnknownHostException {
//construct the history url for job
String addr = getJHSWebappURLWithoutScheme(conf);
- Iterator<String> it = ADDR_SPLITTER.split(addr).iterator();
- it.next(); // ignore the bind host
- String port = it.next();
+ String port;
+ try{
+ Iterator<String> it = ADDR_SPLITTER.split(addr).iterator();
+ it.next(); // ignore the bind host
+ port = it.next();
+ } catch(NoSuchElementException e) {
+ throw new IllegalArgumentException("MapReduce JobHistory WebApp Address"
+ + " does not contain a valid host:port authority: " + addr);
+ }
// Use hs address to figure out the host for webapp
addr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/639f98cc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 96b4e84..1e33b12 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.util.ApplicationClassLoader;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -527,4 +528,12 @@ public class TestMRApps {
assertFalse("/fake/Klass must not be a system class",
ApplicationClassLoader.isSystemClass("/fake/Klass", systemClasses));
}
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidWebappAddress() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, "19888");
+ MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(
+ conf, ApplicationId.newInstance(0, 1));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org